1 //
2 // Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23 //
24
25 // X86 AMD64 Architecture Description File
26
27 //----------REGISTER DEFINITION BLOCK------------------------------------------
28 // This information is used by the matcher and the register allocator to
29 // describe individual registers and classes of registers within the target
30 // architecture.
31
32 register %{
33 //----------Architecture Description Register Definitions----------------------
34 // General Registers
35 // "reg_def" name ( register save type, C convention save type,
36 // ideal register type, encoding );
37 // Register Save Types:
38 //
39 // NS = No-Save: The register allocator assumes that these registers
40 // can be used without saving upon entry to the method, &
41 // that they do not need to be saved at call sites.
42 //
43 // SOC = Save-On-Call: The register allocator assumes that these registers
44 // can be used without saving upon entry to the method,
45 // but that they must be saved at call sites.
46 //
47 // SOE = Save-On-Entry: The register allocator assumes that these registers
48 // must be saved before using them upon entry to the
49 // method, but they do not need to be saved at call
50 // sites.
51 //
52 // AS = Always-Save: The register allocator assumes that these registers
53 // must be saved before using them upon entry to the
54 // method, & that they must be saved at call sites.
55 //
56 // Ideal Register Type is used to determine how to save & restore a
57 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
58 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
59 //
60 // The encoding number is the actual bit-pattern placed into the opcodes.
61
62 // General Registers
63 // R8-R15 must be encoded with REX. (RSP, RBP, RSI, RDI need REX when
64 // used as byte registers)
65
66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
69
70 reg_def RAX (SOC, SOC, Op_RegI, 0, rax->as_VMReg());
71 reg_def RAX_H(SOC, SOC, Op_RegI, 0, rax->as_VMReg()->next());
72
73 reg_def RCX (SOC, SOC, Op_RegI, 1, rcx->as_VMReg());
74 reg_def RCX_H(SOC, SOC, Op_RegI, 1, rcx->as_VMReg()->next());
75
76 reg_def RDX (SOC, SOC, Op_RegI, 2, rdx->as_VMReg());
77 reg_def RDX_H(SOC, SOC, Op_RegI, 2, rdx->as_VMReg()->next());
78
79 reg_def RBX (SOC, SOE, Op_RegI, 3, rbx->as_VMReg());
80 reg_def RBX_H(SOC, SOE, Op_RegI, 3, rbx->as_VMReg()->next());
81
82 reg_def RSP (NS, NS, Op_RegI, 4, rsp->as_VMReg());
83 reg_def RSP_H(NS, NS, Op_RegI, 4, rsp->as_VMReg()->next());
84
85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
86 reg_def RBP (NS, SOE, Op_RegI, 5, rbp->as_VMReg());
87 reg_def RBP_H(NS, SOE, Op_RegI, 5, rbp->as_VMReg()->next());
88
89 #ifdef _WIN64
90
91 reg_def RSI (SOC, SOE, Op_RegI, 6, rsi->as_VMReg());
92 reg_def RSI_H(SOC, SOE, Op_RegI, 6, rsi->as_VMReg()->next());
93
94 reg_def RDI (SOC, SOE, Op_RegI, 7, rdi->as_VMReg());
95 reg_def RDI_H(SOC, SOE, Op_RegI, 7, rdi->as_VMReg()->next());
96
97 #else
98
99 reg_def RSI (SOC, SOC, Op_RegI, 6, rsi->as_VMReg());
100 reg_def RSI_H(SOC, SOC, Op_RegI, 6, rsi->as_VMReg()->next());
101
102 reg_def RDI (SOC, SOC, Op_RegI, 7, rdi->as_VMReg());
103 reg_def RDI_H(SOC, SOC, Op_RegI, 7, rdi->as_VMReg()->next());
104
105 #endif
106
107 reg_def R8 (SOC, SOC, Op_RegI, 8, r8->as_VMReg());
108 reg_def R8_H (SOC, SOC, Op_RegI, 8, r8->as_VMReg()->next());
109
110 reg_def R9 (SOC, SOC, Op_RegI, 9, r9->as_VMReg());
111 reg_def R9_H (SOC, SOC, Op_RegI, 9, r9->as_VMReg()->next());
112
113 reg_def R10 (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
115
116 reg_def R11 (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
118
119 reg_def R12 (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
121
122 reg_def R13 (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
124
125 reg_def R14 (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
127
128 reg_def R15 (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
130
131 reg_def R16 (SOC, SOC, Op_RegI, 16, r16->as_VMReg());
132 reg_def R16_H(SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
133
134 reg_def R17 (SOC, SOC, Op_RegI, 17, r17->as_VMReg());
135 reg_def R17_H(SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
136
137 reg_def R18 (SOC, SOC, Op_RegI, 18, r18->as_VMReg());
138 reg_def R18_H(SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
139
140 reg_def R19 (SOC, SOC, Op_RegI, 19, r19->as_VMReg());
141 reg_def R19_H(SOC, SOC, Op_RegI, 19, r19->as_VMReg()->next());
142
143 reg_def R20 (SOC, SOC, Op_RegI, 20, r20->as_VMReg());
144 reg_def R20_H(SOC, SOC, Op_RegI, 20, r20->as_VMReg()->next());
145
146 reg_def R21 (SOC, SOC, Op_RegI, 21, r21->as_VMReg());
147 reg_def R21_H(SOC, SOC, Op_RegI, 21, r21->as_VMReg()->next());
148
149 reg_def R22 (SOC, SOC, Op_RegI, 22, r22->as_VMReg());
150 reg_def R22_H(SOC, SOC, Op_RegI, 22, r22->as_VMReg()->next());
151
152 reg_def R23 (SOC, SOC, Op_RegI, 23, r23->as_VMReg());
153 reg_def R23_H(SOC, SOC, Op_RegI, 23, r23->as_VMReg()->next());
154
155 reg_def R24 (SOC, SOC, Op_RegI, 24, r24->as_VMReg());
156 reg_def R24_H(SOC, SOC, Op_RegI, 24, r24->as_VMReg()->next());
157
158 reg_def R25 (SOC, SOC, Op_RegI, 25, r25->as_VMReg());
159 reg_def R25_H(SOC, SOC, Op_RegI, 25, r25->as_VMReg()->next());
160
161 reg_def R26 (SOC, SOC, Op_RegI, 26, r26->as_VMReg());
162 reg_def R26_H(SOC, SOC, Op_RegI, 26, r26->as_VMReg()->next());
163
164 reg_def R27 (SOC, SOC, Op_RegI, 27, r27->as_VMReg());
165 reg_def R27_H(SOC, SOC, Op_RegI, 27, r27->as_VMReg()->next());
166
167 reg_def R28 (SOC, SOC, Op_RegI, 28, r28->as_VMReg());
168 reg_def R28_H(SOC, SOC, Op_RegI, 28, r28->as_VMReg()->next());
169
170 reg_def R29 (SOC, SOC, Op_RegI, 29, r29->as_VMReg());
171 reg_def R29_H(SOC, SOC, Op_RegI, 29, r29->as_VMReg()->next());
172
173 reg_def R30 (SOC, SOC, Op_RegI, 30, r30->as_VMReg());
174 reg_def R30_H(SOC, SOC, Op_RegI, 30, r30->as_VMReg()->next());
175
176 reg_def R31 (SOC, SOC, Op_RegI, 31, r31->as_VMReg());
177 reg_def R31_H(SOC, SOC, Op_RegI, 31, r31->as_VMReg()->next());
178
179 // Floating Point Registers
180
181 // Specify priority of register selection within phases of register
182 // allocation. Highest priority is first. A useful heuristic is to
183 // give registers a low priority when they are required by machine
184 // instructions, like EAX and EDX on I486, and choose no-save registers
185 // before save-on-call, & save-on-call before save-on-entry. Registers
186 // which participate in fixed calling sequences should come last.
187 // Registers which are used as pairs must fall on an even boundary.
188
189 alloc_class chunk0(R10, R10_H,
190 R11, R11_H,
191 R8, R8_H,
192 R9, R9_H,
193 R12, R12_H,
194 RCX, RCX_H,
195 RBX, RBX_H,
196 RDI, RDI_H,
197 RDX, RDX_H,
198 RSI, RSI_H,
199 RAX, RAX_H,
200 RBP, RBP_H,
201 R13, R13_H,
202 R14, R14_H,
203 R15, R15_H,
204 R16, R16_H,
205 R17, R17_H,
206 R18, R18_H,
207 R19, R19_H,
208 R20, R20_H,
209 R21, R21_H,
210 R22, R22_H,
211 R23, R23_H,
212 R24, R24_H,
213 R25, R25_H,
214 R26, R26_H,
215 R27, R27_H,
216 R28, R28_H,
217 R29, R29_H,
218 R30, R30_H,
219 R31, R31_H,
220 RSP, RSP_H);
221
222 // XMM registers. 512-bit registers or 8 words each, labeled (a)-p.
223 // Word a in each register holds a Float, words ab hold a Double.
224 // The whole registers are used in SSE4.2 version intrinsics,
225 // array copy stubs and superword operations (see UseSSE42Intrinsics,
226 // UseXMMForArrayCopy and UseSuperword flags).
227 // For pre EVEX enabled architectures:
228 // XMM8-XMM15 must be encoded with REX (VEX for UseAVX)
229 // For EVEX enabled architectures:
230 // XMM8-XMM31 must be encoded with REX (EVEX for UseAVX).
231 //
232 // Linux ABI: No register preserved across function calls
233 // XMM0-XMM7 might hold parameters
234 // Windows ABI: XMM6-XMM15 preserved across function calls
235 // XMM0-XMM3 might hold parameters
236
237 reg_def XMM0 ( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg());
238 reg_def XMM0b( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(1));
239 reg_def XMM0c( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(2));
240 reg_def XMM0d( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(3));
241 reg_def XMM0e( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(4));
242 reg_def XMM0f( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(5));
243 reg_def XMM0g( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(6));
244 reg_def XMM0h( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(7));
245 reg_def XMM0i( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(8));
246 reg_def XMM0j( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(9));
247 reg_def XMM0k( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(10));
248 reg_def XMM0l( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(11));
249 reg_def XMM0m( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(12));
250 reg_def XMM0n( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(13));
251 reg_def XMM0o( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(14));
252 reg_def XMM0p( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(15));
253
254 reg_def XMM1 ( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg());
255 reg_def XMM1b( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(1));
256 reg_def XMM1c( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(2));
257 reg_def XMM1d( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(3));
258 reg_def XMM1e( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(4));
259 reg_def XMM1f( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(5));
260 reg_def XMM1g( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(6));
261 reg_def XMM1h( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(7));
262 reg_def XMM1i( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(8));
263 reg_def XMM1j( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(9));
264 reg_def XMM1k( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(10));
265 reg_def XMM1l( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(11));
266 reg_def XMM1m( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(12));
267 reg_def XMM1n( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(13));
268 reg_def XMM1o( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(14));
269 reg_def XMM1p( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(15));
270
271 reg_def XMM2 ( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg());
272 reg_def XMM2b( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(1));
273 reg_def XMM2c( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(2));
274 reg_def XMM2d( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(3));
275 reg_def XMM2e( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(4));
276 reg_def XMM2f( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(5));
277 reg_def XMM2g( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(6));
278 reg_def XMM2h( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(7));
279 reg_def XMM2i( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(8));
280 reg_def XMM2j( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(9));
281 reg_def XMM2k( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(10));
282 reg_def XMM2l( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(11));
283 reg_def XMM2m( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(12));
284 reg_def XMM2n( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(13));
285 reg_def XMM2o( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(14));
286 reg_def XMM2p( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(15));
287
288 reg_def XMM3 ( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg());
289 reg_def XMM3b( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(1));
290 reg_def XMM3c( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(2));
291 reg_def XMM3d( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(3));
292 reg_def XMM3e( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(4));
293 reg_def XMM3f( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(5));
294 reg_def XMM3g( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(6));
295 reg_def XMM3h( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(7));
296 reg_def XMM3i( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(8));
297 reg_def XMM3j( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(9));
298 reg_def XMM3k( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(10));
299 reg_def XMM3l( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(11));
300 reg_def XMM3m( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(12));
301 reg_def XMM3n( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(13));
302 reg_def XMM3o( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(14));
303 reg_def XMM3p( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(15));
304
305 reg_def XMM4 ( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg());
306 reg_def XMM4b( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(1));
307 reg_def XMM4c( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(2));
308 reg_def XMM4d( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(3));
309 reg_def XMM4e( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(4));
310 reg_def XMM4f( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(5));
311 reg_def XMM4g( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(6));
312 reg_def XMM4h( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(7));
313 reg_def XMM4i( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(8));
314 reg_def XMM4j( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(9));
315 reg_def XMM4k( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(10));
316 reg_def XMM4l( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(11));
317 reg_def XMM4m( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(12));
318 reg_def XMM4n( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(13));
319 reg_def XMM4o( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(14));
320 reg_def XMM4p( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(15));
321
322 reg_def XMM5 ( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg());
323 reg_def XMM5b( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(1));
324 reg_def XMM5c( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(2));
325 reg_def XMM5d( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(3));
326 reg_def XMM5e( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(4));
327 reg_def XMM5f( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(5));
328 reg_def XMM5g( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(6));
329 reg_def XMM5h( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(7));
330 reg_def XMM5i( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(8));
331 reg_def XMM5j( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(9));
332 reg_def XMM5k( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(10));
333 reg_def XMM5l( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(11));
334 reg_def XMM5m( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(12));
335 reg_def XMM5n( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(13));
336 reg_def XMM5o( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(14));
337 reg_def XMM5p( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(15));
338
339 reg_def XMM6 ( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg());
340 reg_def XMM6b( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(1));
341 reg_def XMM6c( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(2));
342 reg_def XMM6d( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(3));
343 reg_def XMM6e( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(4));
344 reg_def XMM6f( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(5));
345 reg_def XMM6g( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(6));
346 reg_def XMM6h( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(7));
347 reg_def XMM6i( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(8));
348 reg_def XMM6j( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(9));
349 reg_def XMM6k( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(10));
350 reg_def XMM6l( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(11));
351 reg_def XMM6m( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(12));
352 reg_def XMM6n( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(13));
353 reg_def XMM6o( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(14));
354 reg_def XMM6p( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(15));
355
356 reg_def XMM7 ( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg());
357 reg_def XMM7b( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(1));
358 reg_def XMM7c( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(2));
359 reg_def XMM7d( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(3));
360 reg_def XMM7e( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(4));
361 reg_def XMM7f( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(5));
362 reg_def XMM7g( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(6));
363 reg_def XMM7h( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(7));
364 reg_def XMM7i( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(8));
365 reg_def XMM7j( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(9));
366 reg_def XMM7k( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(10));
367 reg_def XMM7l( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(11));
368 reg_def XMM7m( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(12));
369 reg_def XMM7n( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(13));
370 reg_def XMM7o( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(14));
371 reg_def XMM7p( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(15));
372
373 reg_def XMM8 ( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg());
374 reg_def XMM8b( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(1));
375 reg_def XMM8c( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(2));
376 reg_def XMM8d( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(3));
377 reg_def XMM8e( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(4));
378 reg_def XMM8f( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(5));
379 reg_def XMM8g( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(6));
380 reg_def XMM8h( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(7));
381 reg_def XMM8i( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(8));
382 reg_def XMM8j( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(9));
383 reg_def XMM8k( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(10));
384 reg_def XMM8l( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(11));
385 reg_def XMM8m( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(12));
386 reg_def XMM8n( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(13));
387 reg_def XMM8o( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(14));
388 reg_def XMM8p( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(15));
389
390 reg_def XMM9 ( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg());
391 reg_def XMM9b( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(1));
392 reg_def XMM9c( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(2));
393 reg_def XMM9d( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(3));
394 reg_def XMM9e( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(4));
395 reg_def XMM9f( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(5));
396 reg_def XMM9g( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(6));
397 reg_def XMM9h( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(7));
398 reg_def XMM9i( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(8));
399 reg_def XMM9j( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(9));
400 reg_def XMM9k( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(10));
401 reg_def XMM9l( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(11));
402 reg_def XMM9m( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(12));
403 reg_def XMM9n( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(13));
404 reg_def XMM9o( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(14));
405 reg_def XMM9p( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(15));
406
407 reg_def XMM10 ( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg());
408 reg_def XMM10b( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(1));
409 reg_def XMM10c( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(2));
410 reg_def XMM10d( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(3));
411 reg_def XMM10e( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(4));
412 reg_def XMM10f( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(5));
413 reg_def XMM10g( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(6));
414 reg_def XMM10h( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(7));
415 reg_def XMM10i( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(8));
416 reg_def XMM10j( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(9));
417 reg_def XMM10k( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(10));
418 reg_def XMM10l( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(11));
419 reg_def XMM10m( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(12));
420 reg_def XMM10n( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(13));
421 reg_def XMM10o( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(14));
422 reg_def XMM10p( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(15));
423
424 reg_def XMM11 ( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg());
425 reg_def XMM11b( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(1));
426 reg_def XMM11c( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(2));
427 reg_def XMM11d( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(3));
428 reg_def XMM11e( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(4));
429 reg_def XMM11f( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(5));
430 reg_def XMM11g( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(6));
431 reg_def XMM11h( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(7));
432 reg_def XMM11i( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(8));
433 reg_def XMM11j( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(9));
434 reg_def XMM11k( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(10));
435 reg_def XMM11l( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(11));
436 reg_def XMM11m( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(12));
437 reg_def XMM11n( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(13));
438 reg_def XMM11o( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(14));
439 reg_def XMM11p( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(15));
440
441 reg_def XMM12 ( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg());
442 reg_def XMM12b( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(1));
443 reg_def XMM12c( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(2));
444 reg_def XMM12d( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(3));
445 reg_def XMM12e( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(4));
446 reg_def XMM12f( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(5));
447 reg_def XMM12g( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(6));
448 reg_def XMM12h( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(7));
449 reg_def XMM12i( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(8));
450 reg_def XMM12j( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(9));
451 reg_def XMM12k( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(10));
452 reg_def XMM12l( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(11));
453 reg_def XMM12m( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(12));
454 reg_def XMM12n( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(13));
455 reg_def XMM12o( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(14));
456 reg_def XMM12p( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(15));
457
458 reg_def XMM13 ( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg());
459 reg_def XMM13b( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(1));
460 reg_def XMM13c( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(2));
461 reg_def XMM13d( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(3));
462 reg_def XMM13e( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(4));
463 reg_def XMM13f( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(5));
464 reg_def XMM13g( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(6));
465 reg_def XMM13h( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(7));
466 reg_def XMM13i( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(8));
467 reg_def XMM13j( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(9));
468 reg_def XMM13k( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(10));
469 reg_def XMM13l( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(11));
470 reg_def XMM13m( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(12));
471 reg_def XMM13n( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(13));
472 reg_def XMM13o( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(14));
473 reg_def XMM13p( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(15));
474
475 reg_def XMM14 ( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg());
476 reg_def XMM14b( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(1));
477 reg_def XMM14c( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(2));
478 reg_def XMM14d( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(3));
479 reg_def XMM14e( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(4));
480 reg_def XMM14f( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(5));
481 reg_def XMM14g( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(6));
482 reg_def XMM14h( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(7));
483 reg_def XMM14i( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(8));
484 reg_def XMM14j( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(9));
485 reg_def XMM14k( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(10));
486 reg_def XMM14l( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(11));
487 reg_def XMM14m( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(12));
488 reg_def XMM14n( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(13));
489 reg_def XMM14o( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(14));
490 reg_def XMM14p( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(15));
491
492 reg_def XMM15 ( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg());
493 reg_def XMM15b( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(1));
494 reg_def XMM15c( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(2));
495 reg_def XMM15d( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(3));
496 reg_def XMM15e( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(4));
497 reg_def XMM15f( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(5));
498 reg_def XMM15g( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(6));
499 reg_def XMM15h( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(7));
500 reg_def XMM15i( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(8));
501 reg_def XMM15j( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(9));
502 reg_def XMM15k( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(10));
503 reg_def XMM15l( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(11));
504 reg_def XMM15m( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(12));
505 reg_def XMM15n( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(13));
506 reg_def XMM15o( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(14));
507 reg_def XMM15p( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(15));
508
509 reg_def XMM16 ( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg());
510 reg_def XMM16b( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(1));
511 reg_def XMM16c( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(2));
512 reg_def XMM16d( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(3));
513 reg_def XMM16e( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(4));
514 reg_def XMM16f( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(5));
515 reg_def XMM16g( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(6));
516 reg_def XMM16h( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(7));
517 reg_def XMM16i( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(8));
518 reg_def XMM16j( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(9));
519 reg_def XMM16k( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(10));
520 reg_def XMM16l( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(11));
521 reg_def XMM16m( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(12));
522 reg_def XMM16n( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(13));
523 reg_def XMM16o( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(14));
524 reg_def XMM16p( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(15));
525
526 reg_def XMM17 ( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg());
527 reg_def XMM17b( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(1));
528 reg_def XMM17c( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(2));
529 reg_def XMM17d( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(3));
530 reg_def XMM17e( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(4));
531 reg_def XMM17f( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(5));
532 reg_def XMM17g( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(6));
533 reg_def XMM17h( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(7));
534 reg_def XMM17i( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(8));
535 reg_def XMM17j( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(9));
536 reg_def XMM17k( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(10));
537 reg_def XMM17l( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(11));
538 reg_def XMM17m( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(12));
539 reg_def XMM17n( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(13));
540 reg_def XMM17o( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(14));
541 reg_def XMM17p( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(15));
542
543 reg_def XMM18 ( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg());
544 reg_def XMM18b( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(1));
545 reg_def XMM18c( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(2));
546 reg_def XMM18d( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(3));
547 reg_def XMM18e( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(4));
548 reg_def XMM18f( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(5));
549 reg_def XMM18g( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(6));
550 reg_def XMM18h( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(7));
551 reg_def XMM18i( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(8));
552 reg_def XMM18j( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(9));
553 reg_def XMM18k( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(10));
554 reg_def XMM18l( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(11));
555 reg_def XMM18m( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(12));
556 reg_def XMM18n( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(13));
557 reg_def XMM18o( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(14));
558 reg_def XMM18p( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(15));
559
560 reg_def XMM19 ( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg());
561 reg_def XMM19b( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(1));
562 reg_def XMM19c( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(2));
563 reg_def XMM19d( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(3));
564 reg_def XMM19e( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(4));
565 reg_def XMM19f( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(5));
566 reg_def XMM19g( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(6));
567 reg_def XMM19h( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(7));
568 reg_def XMM19i( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(8));
569 reg_def XMM19j( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(9));
570 reg_def XMM19k( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(10));
571 reg_def XMM19l( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(11));
572 reg_def XMM19m( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(12));
573 reg_def XMM19n( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(13));
574 reg_def XMM19o( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(14));
575 reg_def XMM19p( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(15));
576
577 reg_def XMM20 ( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg());
578 reg_def XMM20b( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(1));
579 reg_def XMM20c( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(2));
580 reg_def XMM20d( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(3));
581 reg_def XMM20e( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(4));
582 reg_def XMM20f( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(5));
583 reg_def XMM20g( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(6));
584 reg_def XMM20h( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(7));
585 reg_def XMM20i( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(8));
586 reg_def XMM20j( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(9));
587 reg_def XMM20k( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(10));
588 reg_def XMM20l( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(11));
589 reg_def XMM20m( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(12));
590 reg_def XMM20n( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(13));
591 reg_def XMM20o( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(14));
592 reg_def XMM20p( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(15));
593
594 reg_def XMM21 ( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg());
595 reg_def XMM21b( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(1));
596 reg_def XMM21c( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(2));
597 reg_def XMM21d( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(3));
598 reg_def XMM21e( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(4));
599 reg_def XMM21f( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(5));
600 reg_def XMM21g( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(6));
601 reg_def XMM21h( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(7));
602 reg_def XMM21i( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(8));
603 reg_def XMM21j( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(9));
604 reg_def XMM21k( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(10));
605 reg_def XMM21l( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(11));
606 reg_def XMM21m( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(12));
607 reg_def XMM21n( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(13));
608 reg_def XMM21o( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(14));
609 reg_def XMM21p( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(15));
610
611 reg_def XMM22 ( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg());
612 reg_def XMM22b( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(1));
613 reg_def XMM22c( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(2));
614 reg_def XMM22d( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(3));
615 reg_def XMM22e( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(4));
616 reg_def XMM22f( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(5));
617 reg_def XMM22g( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(6));
618 reg_def XMM22h( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(7));
619 reg_def XMM22i( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(8));
620 reg_def XMM22j( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(9));
621 reg_def XMM22k( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(10));
622 reg_def XMM22l( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(11));
623 reg_def XMM22m( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(12));
624 reg_def XMM22n( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(13));
625 reg_def XMM22o( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(14));
626 reg_def XMM22p( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(15));
627
628 reg_def XMM23 ( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg());
629 reg_def XMM23b( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(1));
630 reg_def XMM23c( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(2));
631 reg_def XMM23d( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(3));
632 reg_def XMM23e( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(4));
633 reg_def XMM23f( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(5));
634 reg_def XMM23g( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(6));
635 reg_def XMM23h( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(7));
636 reg_def XMM23i( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(8));
637 reg_def XMM23j( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(9));
638 reg_def XMM23k( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(10));
639 reg_def XMM23l( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(11));
640 reg_def XMM23m( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(12));
641 reg_def XMM23n( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(13));
642 reg_def XMM23o( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(14));
643 reg_def XMM23p( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(15));
644
645 reg_def XMM24 ( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg());
646 reg_def XMM24b( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(1));
647 reg_def XMM24c( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(2));
648 reg_def XMM24d( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(3));
649 reg_def XMM24e( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(4));
650 reg_def XMM24f( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(5));
651 reg_def XMM24g( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(6));
652 reg_def XMM24h( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(7));
653 reg_def XMM24i( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(8));
654 reg_def XMM24j( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(9));
655 reg_def XMM24k( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(10));
656 reg_def XMM24l( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(11));
657 reg_def XMM24m( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(12));
658 reg_def XMM24n( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(13));
659 reg_def XMM24o( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(14));
660 reg_def XMM24p( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(15));
661
662 reg_def XMM25 ( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg());
663 reg_def XMM25b( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(1));
664 reg_def XMM25c( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(2));
665 reg_def XMM25d( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(3));
666 reg_def XMM25e( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(4));
667 reg_def XMM25f( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(5));
668 reg_def XMM25g( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(6));
669 reg_def XMM25h( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(7));
670 reg_def XMM25i( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(8));
671 reg_def XMM25j( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(9));
672 reg_def XMM25k( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(10));
673 reg_def XMM25l( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(11));
674 reg_def XMM25m( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(12));
675 reg_def XMM25n( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(13));
676 reg_def XMM25o( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(14));
677 reg_def XMM25p( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(15));
678
679 reg_def XMM26 ( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg());
680 reg_def XMM26b( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(1));
681 reg_def XMM26c( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(2));
682 reg_def XMM26d( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(3));
683 reg_def XMM26e( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(4));
684 reg_def XMM26f( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(5));
685 reg_def XMM26g( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(6));
686 reg_def XMM26h( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(7));
687 reg_def XMM26i( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(8));
688 reg_def XMM26j( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(9));
689 reg_def XMM26k( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(10));
690 reg_def XMM26l( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(11));
691 reg_def XMM26m( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(12));
692 reg_def XMM26n( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(13));
693 reg_def XMM26o( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(14));
694 reg_def XMM26p( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(15));
695
696 reg_def XMM27 ( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg());
697 reg_def XMM27b( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(1));
698 reg_def XMM27c( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(2));
699 reg_def XMM27d( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(3));
700 reg_def XMM27e( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(4));
701 reg_def XMM27f( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(5));
702 reg_def XMM27g( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(6));
703 reg_def XMM27h( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(7));
704 reg_def XMM27i( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(8));
705 reg_def XMM27j( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(9));
706 reg_def XMM27k( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(10));
707 reg_def XMM27l( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(11));
708 reg_def XMM27m( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(12));
709 reg_def XMM27n( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(13));
710 reg_def XMM27o( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(14));
711 reg_def XMM27p( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(15));
712
713 reg_def XMM28 ( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg());
714 reg_def XMM28b( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(1));
715 reg_def XMM28c( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(2));
716 reg_def XMM28d( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(3));
717 reg_def XMM28e( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(4));
718 reg_def XMM28f( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(5));
719 reg_def XMM28g( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(6));
720 reg_def XMM28h( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(7));
721 reg_def XMM28i( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(8));
722 reg_def XMM28j( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(9));
723 reg_def XMM28k( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(10));
724 reg_def XMM28l( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(11));
725 reg_def XMM28m( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(12));
726 reg_def XMM28n( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(13));
727 reg_def XMM28o( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(14));
728 reg_def XMM28p( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(15));
729
730 reg_def XMM29 ( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg());
731 reg_def XMM29b( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(1));
732 reg_def XMM29c( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(2));
733 reg_def XMM29d( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(3));
734 reg_def XMM29e( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(4));
735 reg_def XMM29f( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(5));
736 reg_def XMM29g( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(6));
737 reg_def XMM29h( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(7));
738 reg_def XMM29i( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(8));
739 reg_def XMM29j( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(9));
740 reg_def XMM29k( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(10));
741 reg_def XMM29l( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(11));
742 reg_def XMM29m( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(12));
743 reg_def XMM29n( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(13));
744 reg_def XMM29o( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(14));
745 reg_def XMM29p( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(15));
746
747 reg_def XMM30 ( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg());
748 reg_def XMM30b( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(1));
749 reg_def XMM30c( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(2));
750 reg_def XMM30d( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(3));
751 reg_def XMM30e( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(4));
752 reg_def XMM30f( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(5));
753 reg_def XMM30g( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(6));
754 reg_def XMM30h( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(7));
755 reg_def XMM30i( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(8));
756 reg_def XMM30j( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(9));
757 reg_def XMM30k( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(10));
758 reg_def XMM30l( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(11));
759 reg_def XMM30m( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(12));
760 reg_def XMM30n( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(13));
761 reg_def XMM30o( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(14));
762 reg_def XMM30p( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(15));
763
764 reg_def XMM31 ( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg());
765 reg_def XMM31b( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(1));
766 reg_def XMM31c( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(2));
767 reg_def XMM31d( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(3));
768 reg_def XMM31e( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(4));
769 reg_def XMM31f( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(5));
770 reg_def XMM31g( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(6));
771 reg_def XMM31h( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(7));
772 reg_def XMM31i( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(8));
773 reg_def XMM31j( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(9));
774 reg_def XMM31k( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(10));
775 reg_def XMM31l( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(11));
776 reg_def XMM31m( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(12));
777 reg_def XMM31n( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(13));
778 reg_def XMM31o( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(14));
779 reg_def XMM31p( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(15));
780
781 reg_def RFLAGS(SOC, SOC, 0, 16, VMRegImpl::Bad());
782
783 // AVX3 Mask Registers.
784 reg_def K1 (SOC, SOC, Op_RegI, 1, k1->as_VMReg());
785 reg_def K1_H (SOC, SOC, Op_RegI, 1, k1->as_VMReg()->next());
786
787 reg_def K2 (SOC, SOC, Op_RegI, 2, k2->as_VMReg());
788 reg_def K2_H (SOC, SOC, Op_RegI, 2, k2->as_VMReg()->next());
789
790 reg_def K3 (SOC, SOC, Op_RegI, 3, k3->as_VMReg());
791 reg_def K3_H (SOC, SOC, Op_RegI, 3, k3->as_VMReg()->next());
792
793 reg_def K4 (SOC, SOC, Op_RegI, 4, k4->as_VMReg());
794 reg_def K4_H (SOC, SOC, Op_RegI, 4, k4->as_VMReg()->next());
795
796 reg_def K5 (SOC, SOC, Op_RegI, 5, k5->as_VMReg());
797 reg_def K5_H (SOC, SOC, Op_RegI, 5, k5->as_VMReg()->next());
798
799 reg_def K6 (SOC, SOC, Op_RegI, 6, k6->as_VMReg());
800 reg_def K6_H (SOC, SOC, Op_RegI, 6, k6->as_VMReg()->next());
801
802 reg_def K7 (SOC, SOC, Op_RegI, 7, k7->as_VMReg());
803 reg_def K7_H (SOC, SOC, Op_RegI, 7, k7->as_VMReg()->next());
804
805
806 //----------Architecture Description Register Classes--------------------------
807 // Several register classes are automatically defined based upon information in
808 // this architecture description.
809 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
810 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
811 //
812
813 // Empty register class.
814 reg_class no_reg();
815
816 // Class for all pointer/long registers including APX extended GPRs.
817 reg_class all_reg(RAX, RAX_H,
818 RDX, RDX_H,
819 RBP, RBP_H,
820 RDI, RDI_H,
821 RSI, RSI_H,
822 RCX, RCX_H,
823 RBX, RBX_H,
824 RSP, RSP_H,
825 R8, R8_H,
826 R9, R9_H,
827 R10, R10_H,
828 R11, R11_H,
829 R12, R12_H,
830 R13, R13_H,
831 R14, R14_H,
832 R15, R15_H,
833 R16, R16_H,
834 R17, R17_H,
835 R18, R18_H,
836 R19, R19_H,
837 R20, R20_H,
838 R21, R21_H,
839 R22, R22_H,
840 R23, R23_H,
841 R24, R24_H,
842 R25, R25_H,
843 R26, R26_H,
844 R27, R27_H,
845 R28, R28_H,
846 R29, R29_H,
847 R30, R30_H,
848 R31, R31_H);
849
850 // Class for all int registers including APX extended GPRs.
851 reg_class all_int_reg(RAX
852 RDX,
853 RBP,
854 RDI,
855 RSI,
856 RCX,
857 RBX,
858 R8,
859 R9,
860 R10,
861 R11,
862 R12,
863 R13,
864 R14,
865 R16,
866 R17,
867 R18,
868 R19,
869 R20,
870 R21,
871 R22,
872 R23,
873 R24,
874 R25,
875 R26,
876 R27,
877 R28,
878 R29,
879 R30,
880 R31);
881
882 // Class for all pointer registers
883 reg_class any_reg %{
884 return _ANY_REG_mask;
885 %}
886
887 // Class for all pointer registers (excluding RSP)
888 reg_class ptr_reg %{
889 return _PTR_REG_mask;
890 %}
891
892 // Class for all pointer registers (excluding RSP and RBP)
893 reg_class ptr_reg_no_rbp %{
894 return _PTR_REG_NO_RBP_mask;
895 %}
896
897 // Class for all pointer registers (excluding RAX and RSP)
898 reg_class ptr_no_rax_reg %{
899 return _PTR_NO_RAX_REG_mask;
900 %}
901
902 // Class for all pointer registers (excluding RAX, RBX, and RSP)
903 reg_class ptr_no_rax_rbx_reg %{
904 return _PTR_NO_RAX_RBX_REG_mask;
905 %}
906
907 // Class for all long registers (excluding RSP)
908 reg_class long_reg %{
909 return _LONG_REG_mask;
910 %}
911
912 // Class for all long registers (excluding RAX, RDX and RSP)
913 reg_class long_no_rax_rdx_reg %{
914 return _LONG_NO_RAX_RDX_REG_mask;
915 %}
916
917 // Class for all long registers (excluding RCX and RSP)
918 reg_class long_no_rcx_reg %{
919 return _LONG_NO_RCX_REG_mask;
920 %}
921
922 // Class for all long registers (excluding RBP and R13)
923 reg_class long_no_rbp_r13_reg %{
924 return _LONG_NO_RBP_R13_REG_mask;
925 %}
926
927 // Class for all int registers (excluding RSP)
928 reg_class int_reg %{
929 return _INT_REG_mask;
930 %}
931
932 // Class for all int registers (excluding RAX, RDX, and RSP)
933 reg_class int_no_rax_rdx_reg %{
934 return _INT_NO_RAX_RDX_REG_mask;
935 %}
936
937 // Class for all int registers (excluding RCX and RSP)
938 reg_class int_no_rcx_reg %{
939 return _INT_NO_RCX_REG_mask;
940 %}
941
942 // Class for all int registers (excluding RBP and R13)
943 reg_class int_no_rbp_r13_reg %{
944 return _INT_NO_RBP_R13_REG_mask;
945 %}
946
947 // Singleton class for RAX pointer register
948 reg_class ptr_rax_reg(RAX, RAX_H);
949
950 // Singleton class for RBX pointer register
951 reg_class ptr_rbx_reg(RBX, RBX_H);
952
953 // Singleton class for RSI pointer register
954 reg_class ptr_rsi_reg(RSI, RSI_H);
955
956 // Singleton class for RBP pointer register
957 reg_class ptr_rbp_reg(RBP, RBP_H);
958
959 // Singleton class for RDI pointer register
960 reg_class ptr_rdi_reg(RDI, RDI_H);
961
962 // Singleton class for stack pointer
963 reg_class ptr_rsp_reg(RSP, RSP_H);
964
965 // Singleton class for TLS pointer
966 reg_class ptr_r15_reg(R15, R15_H);
967
968 // Singleton class for RAX long register
969 reg_class long_rax_reg(RAX, RAX_H);
970
971 // Singleton class for RCX long register
972 reg_class long_rcx_reg(RCX, RCX_H);
973
974 // Singleton class for RDX long register
975 reg_class long_rdx_reg(RDX, RDX_H);
976
977 // Singleton class for R11 long register
978 reg_class long_r11_reg(R11, R11_H);
979
980 // Singleton class for RAX int register
981 reg_class int_rax_reg(RAX);
982
983 // Singleton class for RBX int register
984 reg_class int_rbx_reg(RBX);
985
986 // Singleton class for RCX int register
987 reg_class int_rcx_reg(RCX);
988
989 // Singleton class for RDX int register
990 reg_class int_rdx_reg(RDX);
991
992 // Singleton class for RDI int register
993 reg_class int_rdi_reg(RDI);
994
995 // Singleton class for instruction pointer
996 // reg_class ip_reg(RIP);
997
998 alloc_class chunk1(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p,
999 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p,
1000 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p,
1001 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p,
1002 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p,
1003 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p,
1004 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p,
1005 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p,
1006 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p,
1007 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p,
1008 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
1009 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
1010 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
1011 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
1012 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
1013 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p,
1014 XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p,
1015 XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p,
1016 XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p,
1017 XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p,
1018 XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p,
1019 XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p,
1020 XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p,
1021 XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p,
1022 XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p,
1023 XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p,
1024 XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p,
1025 XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p,
1026 XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p,
1027 XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p,
1028 XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p,
1029 XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p);
1030
1031 alloc_class chunk2(K7, K7_H,
1032 K6, K6_H,
1033 K5, K5_H,
1034 K4, K4_H,
1035 K3, K3_H,
1036 K2, K2_H,
1037 K1, K1_H);
1038
1039 reg_class vectmask_reg(K1, K1_H,
1040 K2, K2_H,
1041 K3, K3_H,
1042 K4, K4_H,
1043 K5, K5_H,
1044 K6, K6_H,
1045 K7, K7_H);
1046
1047 reg_class vectmask_reg_K1(K1, K1_H);
1048 reg_class vectmask_reg_K2(K2, K2_H);
1049 reg_class vectmask_reg_K3(K3, K3_H);
1050 reg_class vectmask_reg_K4(K4, K4_H);
1051 reg_class vectmask_reg_K5(K5, K5_H);
1052 reg_class vectmask_reg_K6(K6, K6_H);
1053 reg_class vectmask_reg_K7(K7, K7_H);
1054
1055 // flags allocation class should be last.
1056 alloc_class chunk3(RFLAGS);
1057
1058 // Singleton class for condition codes
1059 reg_class int_flags(RFLAGS);
1060
1061 // Class for pre evex float registers
1062 reg_class float_reg_legacy(XMM0,
1063 XMM1,
1064 XMM2,
1065 XMM3,
1066 XMM4,
1067 XMM5,
1068 XMM6,
1069 XMM7,
1070 XMM8,
1071 XMM9,
1072 XMM10,
1073 XMM11,
1074 XMM12,
1075 XMM13,
1076 XMM14,
1077 XMM15);
1078
1079 // Class for evex float registers
1080 reg_class float_reg_evex(XMM0,
1081 XMM1,
1082 XMM2,
1083 XMM3,
1084 XMM4,
1085 XMM5,
1086 XMM6,
1087 XMM7,
1088 XMM8,
1089 XMM9,
1090 XMM10,
1091 XMM11,
1092 XMM12,
1093 XMM13,
1094 XMM14,
1095 XMM15,
1096 XMM16,
1097 XMM17,
1098 XMM18,
1099 XMM19,
1100 XMM20,
1101 XMM21,
1102 XMM22,
1103 XMM23,
1104 XMM24,
1105 XMM25,
1106 XMM26,
1107 XMM27,
1108 XMM28,
1109 XMM29,
1110 XMM30,
1111 XMM31);
1112
1113 reg_class_dynamic float_reg(float_reg_evex, float_reg_legacy, %{ VM_Version::supports_evex() %} );
1114 reg_class_dynamic float_reg_vl(float_reg_evex, float_reg_legacy, %{ VM_Version::supports_evex() && VM_Version::supports_avx512vl() %} );
1115
1116 // Class for pre evex double registers
1117 reg_class double_reg_legacy(XMM0, XMM0b,
1118 XMM1, XMM1b,
1119 XMM2, XMM2b,
1120 XMM3, XMM3b,
1121 XMM4, XMM4b,
1122 XMM5, XMM5b,
1123 XMM6, XMM6b,
1124 XMM7, XMM7b,
1125 XMM8, XMM8b,
1126 XMM9, XMM9b,
1127 XMM10, XMM10b,
1128 XMM11, XMM11b,
1129 XMM12, XMM12b,
1130 XMM13, XMM13b,
1131 XMM14, XMM14b,
1132 XMM15, XMM15b);
1133
1134 // Class for evex double registers
1135 reg_class double_reg_evex(XMM0, XMM0b,
1136 XMM1, XMM1b,
1137 XMM2, XMM2b,
1138 XMM3, XMM3b,
1139 XMM4, XMM4b,
1140 XMM5, XMM5b,
1141 XMM6, XMM6b,
1142 XMM7, XMM7b,
1143 XMM8, XMM8b,
1144 XMM9, XMM9b,
1145 XMM10, XMM10b,
1146 XMM11, XMM11b,
1147 XMM12, XMM12b,
1148 XMM13, XMM13b,
1149 XMM14, XMM14b,
1150 XMM15, XMM15b,
1151 XMM16, XMM16b,
1152 XMM17, XMM17b,
1153 XMM18, XMM18b,
1154 XMM19, XMM19b,
1155 XMM20, XMM20b,
1156 XMM21, XMM21b,
1157 XMM22, XMM22b,
1158 XMM23, XMM23b,
1159 XMM24, XMM24b,
1160 XMM25, XMM25b,
1161 XMM26, XMM26b,
1162 XMM27, XMM27b,
1163 XMM28, XMM28b,
1164 XMM29, XMM29b,
1165 XMM30, XMM30b,
1166 XMM31, XMM31b);
1167
1168 reg_class_dynamic double_reg(double_reg_evex, double_reg_legacy, %{ VM_Version::supports_evex() %} );
1169 reg_class_dynamic double_reg_vl(double_reg_evex, double_reg_legacy, %{ VM_Version::supports_evex() && VM_Version::supports_avx512vl() %} );
1170
1171 // Class for pre evex 32bit vector registers
1172 reg_class vectors_reg_legacy(XMM0,
1173 XMM1,
1174 XMM2,
1175 XMM3,
1176 XMM4,
1177 XMM5,
1178 XMM6,
1179 XMM7,
1180 XMM8,
1181 XMM9,
1182 XMM10,
1183 XMM11,
1184 XMM12,
1185 XMM13,
1186 XMM14,
1187 XMM15);
1188
1189 // Class for evex 32bit vector registers
1190 reg_class vectors_reg_evex(XMM0,
1191 XMM1,
1192 XMM2,
1193 XMM3,
1194 XMM4,
1195 XMM5,
1196 XMM6,
1197 XMM7,
1198 XMM8,
1199 XMM9,
1200 XMM10,
1201 XMM11,
1202 XMM12,
1203 XMM13,
1204 XMM14,
1205 XMM15,
1206 XMM16,
1207 XMM17,
1208 XMM18,
1209 XMM19,
1210 XMM20,
1211 XMM21,
1212 XMM22,
1213 XMM23,
1214 XMM24,
1215 XMM25,
1216 XMM26,
1217 XMM27,
1218 XMM28,
1219 XMM29,
1220 XMM30,
1221 XMM31);
1222
1223 reg_class_dynamic vectors_reg(vectors_reg_evex, vectors_reg_legacy, %{ VM_Version::supports_evex() %} );
1224 reg_class_dynamic vectors_reg_vlbwdq(vectors_reg_evex, vectors_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1225
1226 // Class for all 64bit vector registers
1227 reg_class vectord_reg_legacy(XMM0, XMM0b,
1228 XMM1, XMM1b,
1229 XMM2, XMM2b,
1230 XMM3, XMM3b,
1231 XMM4, XMM4b,
1232 XMM5, XMM5b,
1233 XMM6, XMM6b,
1234 XMM7, XMM7b,
1235 XMM8, XMM8b,
1236 XMM9, XMM9b,
1237 XMM10, XMM10b,
1238 XMM11, XMM11b,
1239 XMM12, XMM12b,
1240 XMM13, XMM13b,
1241 XMM14, XMM14b,
1242 XMM15, XMM15b);
1243
1244 // Class for all 64bit vector registers
1245 reg_class vectord_reg_evex(XMM0, XMM0b,
1246 XMM1, XMM1b,
1247 XMM2, XMM2b,
1248 XMM3, XMM3b,
1249 XMM4, XMM4b,
1250 XMM5, XMM5b,
1251 XMM6, XMM6b,
1252 XMM7, XMM7b,
1253 XMM8, XMM8b,
1254 XMM9, XMM9b,
1255 XMM10, XMM10b,
1256 XMM11, XMM11b,
1257 XMM12, XMM12b,
1258 XMM13, XMM13b,
1259 XMM14, XMM14b,
1260 XMM15, XMM15b,
1261 XMM16, XMM16b,
1262 XMM17, XMM17b,
1263 XMM18, XMM18b,
1264 XMM19, XMM19b,
1265 XMM20, XMM20b,
1266 XMM21, XMM21b,
1267 XMM22, XMM22b,
1268 XMM23, XMM23b,
1269 XMM24, XMM24b,
1270 XMM25, XMM25b,
1271 XMM26, XMM26b,
1272 XMM27, XMM27b,
1273 XMM28, XMM28b,
1274 XMM29, XMM29b,
1275 XMM30, XMM30b,
1276 XMM31, XMM31b);
1277
1278 reg_class_dynamic vectord_reg(vectord_reg_evex, vectord_reg_legacy, %{ VM_Version::supports_evex() %} );
1279 reg_class_dynamic vectord_reg_vlbwdq(vectord_reg_evex, vectord_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1280
1281 // Class for all 128bit vector registers
1282 reg_class vectorx_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d,
1283 XMM1, XMM1b, XMM1c, XMM1d,
1284 XMM2, XMM2b, XMM2c, XMM2d,
1285 XMM3, XMM3b, XMM3c, XMM3d,
1286 XMM4, XMM4b, XMM4c, XMM4d,
1287 XMM5, XMM5b, XMM5c, XMM5d,
1288 XMM6, XMM6b, XMM6c, XMM6d,
1289 XMM7, XMM7b, XMM7c, XMM7d,
1290 XMM8, XMM8b, XMM8c, XMM8d,
1291 XMM9, XMM9b, XMM9c, XMM9d,
1292 XMM10, XMM10b, XMM10c, XMM10d,
1293 XMM11, XMM11b, XMM11c, XMM11d,
1294 XMM12, XMM12b, XMM12c, XMM12d,
1295 XMM13, XMM13b, XMM13c, XMM13d,
1296 XMM14, XMM14b, XMM14c, XMM14d,
1297 XMM15, XMM15b, XMM15c, XMM15d);
1298
1299 // Class for all 128bit vector registers
1300 reg_class vectorx_reg_evex(XMM0, XMM0b, XMM0c, XMM0d,
1301 XMM1, XMM1b, XMM1c, XMM1d,
1302 XMM2, XMM2b, XMM2c, XMM2d,
1303 XMM3, XMM3b, XMM3c, XMM3d,
1304 XMM4, XMM4b, XMM4c, XMM4d,
1305 XMM5, XMM5b, XMM5c, XMM5d,
1306 XMM6, XMM6b, XMM6c, XMM6d,
1307 XMM7, XMM7b, XMM7c, XMM7d,
1308 XMM8, XMM8b, XMM8c, XMM8d,
1309 XMM9, XMM9b, XMM9c, XMM9d,
1310 XMM10, XMM10b, XMM10c, XMM10d,
1311 XMM11, XMM11b, XMM11c, XMM11d,
1312 XMM12, XMM12b, XMM12c, XMM12d,
1313 XMM13, XMM13b, XMM13c, XMM13d,
1314 XMM14, XMM14b, XMM14c, XMM14d,
1315 XMM15, XMM15b, XMM15c, XMM15d,
1316 XMM16, XMM16b, XMM16c, XMM16d,
1317 XMM17, XMM17b, XMM17c, XMM17d,
1318 XMM18, XMM18b, XMM18c, XMM18d,
1319 XMM19, XMM19b, XMM19c, XMM19d,
1320 XMM20, XMM20b, XMM20c, XMM20d,
1321 XMM21, XMM21b, XMM21c, XMM21d,
1322 XMM22, XMM22b, XMM22c, XMM22d,
1323 XMM23, XMM23b, XMM23c, XMM23d,
1324 XMM24, XMM24b, XMM24c, XMM24d,
1325 XMM25, XMM25b, XMM25c, XMM25d,
1326 XMM26, XMM26b, XMM26c, XMM26d,
1327 XMM27, XMM27b, XMM27c, XMM27d,
1328 XMM28, XMM28b, XMM28c, XMM28d,
1329 XMM29, XMM29b, XMM29c, XMM29d,
1330 XMM30, XMM30b, XMM30c, XMM30d,
1331 XMM31, XMM31b, XMM31c, XMM31d);
1332
1333 reg_class_dynamic vectorx_reg(vectorx_reg_evex, vectorx_reg_legacy, %{ VM_Version::supports_evex() %} );
1334 reg_class_dynamic vectorx_reg_vlbwdq(vectorx_reg_evex, vectorx_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1335
1336 // Class for all 256bit vector registers
1337 reg_class vectory_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h,
1338 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h,
1339 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h,
1340 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h,
1341 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h,
1342 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h,
1343 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h,
1344 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h,
1345 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h,
1346 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h,
1347 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h,
1348 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h,
1349 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h,
1350 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h,
1351 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h,
1352 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h);
1353
1354 // Class for all 256bit vector registers
1355 reg_class vectory_reg_evex(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h,
1356 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h,
1357 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h,
1358 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h,
1359 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h,
1360 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h,
1361 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h,
1362 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h,
1363 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h,
1364 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h,
1365 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h,
1366 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h,
1367 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h,
1368 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h,
1369 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h,
1370 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h,
1371 XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h,
1372 XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h,
1373 XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h,
1374 XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h,
1375 XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h,
1376 XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h,
1377 XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h,
1378 XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h,
1379 XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h,
1380 XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h,
1381 XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h,
1382 XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h,
1383 XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h,
1384 XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h,
1385 XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h,
1386 XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h);
1387
1388 reg_class_dynamic vectory_reg(vectory_reg_evex, vectory_reg_legacy, %{ VM_Version::supports_evex() %} );
1389 reg_class_dynamic vectory_reg_vlbwdq(vectory_reg_evex, vectory_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1390
1391 // Class for all 512bit vector registers
1392 reg_class vectorz_reg_evex(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p,
1393 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p,
1394 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p,
1395 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p,
1396 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p,
1397 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p,
1398 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p,
1399 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p,
1400 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p,
1401 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p,
1402 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
1403 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
1404 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
1405 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
1406 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
1407 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p,
1408 XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p,
1409 XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p,
1410 XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p,
1411 XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p,
1412 XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p,
1413 XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p,
1414 XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p,
1415 XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p,
1416 XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p,
1417 XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p,
1418 XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p,
1419 XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p,
1420 XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p,
1421 XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p,
1422 XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p,
1423 XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p);
1424
1425 // Class for restricted 512bit vector registers
1426 reg_class vectorz_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p,
1427 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p,
1428 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p,
1429 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p,
1430 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p,
1431 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p,
1432 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p,
1433 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p,
1434 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p,
1435 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p,
1436 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
1437 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
1438 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
1439 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
1440 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
1441 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p);
1442
1443 reg_class_dynamic vectorz_reg (vectorz_reg_evex, vectorz_reg_legacy, %{ VM_Version::supports_evex() %} );
1444 reg_class_dynamic vectorz_reg_vl(vectorz_reg_evex, vectorz_reg_legacy, %{ VM_Version::supports_evex() && VM_Version::supports_avx512vl() %} );
1445
1446 reg_class xmm0_reg(XMM0, XMM0b, XMM0c, XMM0d);
1447
1448 %}
1449
1450
1451 //----------SOURCE BLOCK-------------------------------------------------------
1452 // This is a block of C++ code which provides values, functions, and
1453 // definitions necessary in the rest of the architecture description
1454
1455 source_hpp %{
1456
1457 #include "peephole_x86_64.hpp"
1458
1459 bool castLL_is_imm32(const Node* n);
1460
1461 %}
1462
1463 source %{
1464
1465 bool castLL_is_imm32(const Node* n) {
1466 assert(n->is_CastLL(), "must be a CastLL");
1467 const TypeLong* t = n->bottom_type()->is_long();
1468 return (t->_lo == min_jlong || Assembler::is_simm32(t->_lo)) && (t->_hi == max_jlong || Assembler::is_simm32(t->_hi));
1469 }
1470
1471 %}
1472
1473 // Register masks
1474 source_hpp %{
1475
1476 extern RegMask _ANY_REG_mask;
1477 extern RegMask _PTR_REG_mask;
1478 extern RegMask _PTR_REG_NO_RBP_mask;
1479 extern RegMask _PTR_NO_RAX_REG_mask;
1480 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
1481 extern RegMask _LONG_REG_mask;
1482 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
1483 extern RegMask _LONG_NO_RCX_REG_mask;
1484 extern RegMask _LONG_NO_RBP_R13_REG_mask;
1485 extern RegMask _INT_REG_mask;
1486 extern RegMask _INT_NO_RAX_RDX_REG_mask;
1487 extern RegMask _INT_NO_RCX_REG_mask;
1488 extern RegMask _INT_NO_RBP_R13_REG_mask;
1489 extern RegMask _FLOAT_REG_mask;
1490
1491 extern RegMask _STACK_OR_PTR_REG_mask;
1492 extern RegMask _STACK_OR_LONG_REG_mask;
1493 extern RegMask _STACK_OR_INT_REG_mask;
1494
1495 inline const RegMask& STACK_OR_PTR_REG_mask() { return _STACK_OR_PTR_REG_mask; }
1496 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
1497 inline const RegMask& STACK_OR_INT_REG_mask() { return _STACK_OR_INT_REG_mask; }
1498
1499 %}
1500
1501 source %{
1502 #define RELOC_IMM64 Assembler::imm_operand
1503 #define RELOC_DISP32 Assembler::disp32_operand
1504
1505 #define __ masm->
1506
1507 RegMask _ANY_REG_mask;
1508 RegMask _PTR_REG_mask;
1509 RegMask _PTR_REG_NO_RBP_mask;
1510 RegMask _PTR_NO_RAX_REG_mask;
1511 RegMask _PTR_NO_RAX_RBX_REG_mask;
1512 RegMask _LONG_REG_mask;
1513 RegMask _LONG_NO_RAX_RDX_REG_mask;
1514 RegMask _LONG_NO_RCX_REG_mask;
1515 RegMask _LONG_NO_RBP_R13_REG_mask;
1516 RegMask _INT_REG_mask;
1517 RegMask _INT_NO_RAX_RDX_REG_mask;
1518 RegMask _INT_NO_RCX_REG_mask;
1519 RegMask _INT_NO_RBP_R13_REG_mask;
1520 RegMask _FLOAT_REG_mask;
1521 RegMask _STACK_OR_PTR_REG_mask;
1522 RegMask _STACK_OR_LONG_REG_mask;
1523 RegMask _STACK_OR_INT_REG_mask;
1524
1525 static bool need_r12_heapbase() {
1526 return UseCompressedOops;
1527 }
1528
1529 void reg_mask_init() {
1530 constexpr Register egprs[] = {r16, r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31};
1531
1532 // _ALL_REG_mask is generated by adlc from the all_reg register class below.
1533 // We derive a number of subsets from it.
1534 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1535
1536 if (PreserveFramePointer) {
1537 _ANY_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1538 _ANY_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
1539 }
1540 if (need_r12_heapbase()) {
1541 _ANY_REG_mask.remove(OptoReg::as_OptoReg(r12->as_VMReg()));
1542 _ANY_REG_mask.remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
1543 }
1544
1545 _PTR_REG_mask.assignFrom(_ANY_REG_mask);
1546 _PTR_REG_mask.remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
1547 _PTR_REG_mask.remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
1548 _PTR_REG_mask.remove(OptoReg::as_OptoReg(r15->as_VMReg()));
1549 _PTR_REG_mask.remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
1550 if (!UseAPX) {
1551 for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
1552 _PTR_REG_mask.remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
1553 _PTR_REG_mask.remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()->next()));
1554 }
1555 }
1556
1557 _STACK_OR_PTR_REG_mask.assignFrom(_PTR_REG_mask);
1558 _STACK_OR_PTR_REG_mask.or_with(STACK_OR_STACK_SLOTS_mask());
1559
1560 _PTR_REG_NO_RBP_mask.assignFrom(_PTR_REG_mask);
1561 _PTR_REG_NO_RBP_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1562 _PTR_REG_NO_RBP_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
1563
1564 _PTR_NO_RAX_REG_mask.assignFrom(_PTR_REG_mask);
1565 _PTR_NO_RAX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()));
1566 _PTR_NO_RAX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
1567
1568 _PTR_NO_RAX_RBX_REG_mask.assignFrom(_PTR_NO_RAX_REG_mask);
1569 _PTR_NO_RAX_RBX_REG_mask.remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
1570 _PTR_NO_RAX_RBX_REG_mask.remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
1571
1572
1573 _LONG_REG_mask.assignFrom(_PTR_REG_mask);
1574 _STACK_OR_LONG_REG_mask.assignFrom(_LONG_REG_mask);
1575 _STACK_OR_LONG_REG_mask.or_with(STACK_OR_STACK_SLOTS_mask());
1576
1577 _LONG_NO_RAX_RDX_REG_mask.assignFrom(_LONG_REG_mask);
1578 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()));
1579 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
1580 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
1581 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
1582
1583 _LONG_NO_RCX_REG_mask.assignFrom(_LONG_REG_mask);
1584 _LONG_NO_RCX_REG_mask.remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
1585 _LONG_NO_RCX_REG_mask.remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
1586
1587 _LONG_NO_RBP_R13_REG_mask.assignFrom(_LONG_REG_mask);
1588 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1589 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
1590 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(r13->as_VMReg()));
1591 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
1592
1593 _INT_REG_mask.assignFrom(_ALL_INT_REG_mask);
1594 if (!UseAPX) {
1595 for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
1596 _INT_REG_mask.remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
1597 }
1598 }
1599
1600 if (PreserveFramePointer) {
1601 _INT_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1602 }
1603 if (need_r12_heapbase()) {
1604 _INT_REG_mask.remove(OptoReg::as_OptoReg(r12->as_VMReg()));
1605 }
1606
1607 _STACK_OR_INT_REG_mask.assignFrom(_INT_REG_mask);
1608 _STACK_OR_INT_REG_mask.or_with(STACK_OR_STACK_SLOTS_mask());
1609
1610 _INT_NO_RAX_RDX_REG_mask.assignFrom(_INT_REG_mask);
1611 _INT_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()));
1612 _INT_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
1613
1614 _INT_NO_RCX_REG_mask.assignFrom(_INT_REG_mask);
1615 _INT_NO_RCX_REG_mask.remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
1616
1617 _INT_NO_RBP_R13_REG_mask.assignFrom(_INT_REG_mask);
1618 _INT_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1619 _INT_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(r13->as_VMReg()));
1620
1621 // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
1622 // from the float_reg_legacy/float_reg_evex register class.
1623 _FLOAT_REG_mask.assignFrom(VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask);
1624 }
1625
1626 static bool generate_vzeroupper(Compile* C) {
1627 return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false; // Generate vzeroupper
1628 }
1629
1630 static int clear_avx_size() {
1631 return generate_vzeroupper(Compile::current()) ? 3: 0; // vzeroupper
1632 }
1633
1634 // !!!!! Special hack to get all types of calls to specify the byte offset
1635 // from the start of the call to the point where the return address
1636 // will point.
1637 int MachCallStaticJavaNode::ret_addr_offset() const
1638 {
1639 int offset = 5; // 5 bytes from start of call to where return address points
1640 offset += clear_avx_size();
1641 return offset;
1642 }
1643
1644 int MachCallDynamicJavaNode::ret_addr_offset() const
1645 {
1646 int offset = 15; // 15 bytes from start of call to where return address points
1647 offset += clear_avx_size();
1648 return offset;
1649 }
1650
1651 int MachCallRuntimeNode::ret_addr_offset() const {
1652 if (_entry_point == nullptr) {
1653 // CallLeafNoFPInDirect
1654 return 3; // callq (register)
1655 }
1656 int offset = 13; // movq r10,#addr; callq (r10)
1657 if (this->ideal_Opcode() != Op_CallLeafVector) {
1658 offset += clear_avx_size();
1659 }
1660 return offset;
1661 }
1662 //
1663 // Compute padding required for nodes which need alignment
1664 //
1665
1666 // The address of the call instruction needs to be 4-byte aligned to
1667 // ensure that it does not span a cache line so that it can be patched.
1668 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
1669 {
1670 current_offset += clear_avx_size(); // skip vzeroupper
1671 current_offset += 1; // skip call opcode byte
1672 return align_up(current_offset, alignment_required()) - current_offset;
1673 }
1674
1675 // The address of the call instruction needs to be 4-byte aligned to
1676 // ensure that it does not span a cache line so that it can be patched.
1677 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
1678 {
1679 current_offset += clear_avx_size(); // skip vzeroupper
1680 current_offset += 11; // skip movq instruction + call opcode byte
1681 return align_up(current_offset, alignment_required()) - current_offset;
1682 }
1683
1684 // This could be in MacroAssembler but it's fairly C2 specific
1685 static void emit_cmpfp_fixup(MacroAssembler* masm) {
1686 Label exit;
1687 __ jccb(Assembler::noParity, exit);
1688 __ pushf();
1689 //
1690 // comiss/ucomiss instructions set ZF,PF,CF flags and
1691 // zero OF,AF,SF for NaN values.
1692 // Fixup flags by zeroing ZF,PF so that compare of NaN
1693 // values returns 'less than' result (CF is set).
1694 // Leave the rest of flags unchanged.
1695 //
1696 // 7 6 5 4 3 2 1 0
1697 // |S|Z|r|A|r|P|r|C| (r - reserved bit)
1698 // 0 0 1 0 1 0 1 1 (0x2B)
1699 //
1700 __ andq(Address(rsp, 0), 0xffffff2b);
1701 __ popf();
1702 __ bind(exit);
1703 }
1704
1705 static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
1706 // If any floating point comparison instruction is used, unordered case always triggers jump
1707 // for below condition, CF=1 is true when at least one input is NaN
1708 Label done;
1709 __ movl(dst, -1);
1710 __ jcc(Assembler::below, done);
1711 __ setcc(Assembler::notEqual, dst);
1712 __ bind(done);
1713 }
1714
1715 enum FP_PREC {
1716 fp_prec_hlf,
1717 fp_prec_flt,
1718 fp_prec_dbl
1719 };
1720
1721 static inline void emit_fp_ucom(MacroAssembler* masm, enum FP_PREC pt,
1722 XMMRegister p, XMMRegister q) {
1723 if (pt == fp_prec_hlf) {
1724 __ evucomish(p, q);
1725 } else if (pt == fp_prec_flt) {
1726 __ ucomiss(p, q);
1727 } else {
1728 __ ucomisd(p, q);
1729 }
1730 }
1731
1732 static inline void movfp(MacroAssembler* masm, enum FP_PREC pt,
1733 XMMRegister dst, XMMRegister src, Register scratch) {
1734 if (pt == fp_prec_hlf) {
1735 __ movhlf(dst, src, scratch);
1736 } else if (pt == fp_prec_flt) {
1737 __ movflt(dst, src);
1738 } else {
1739 __ movdbl(dst, src);
1740 }
1741 }
1742
1743 // Math.min() # Math.max()
1744 // -----------------------------
1745 // (v)ucomis[h/s/d] #
1746 // ja -> b # a
1747 // jp -> NaN # NaN
1748 // jb -> a # b
1749 // je -> a | b # a & b
1750 static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
1751 XMMRegister a, XMMRegister b, Register rt,
1752 bool min, enum FP_PREC pt) {
1753 Label nan, zero, below, above, done;
1754
1755 emit_fp_ucom(masm, pt, a, b);
1756
1757 if (dst->encoding() != (min ? b : a)->encoding()) {
1758 __ jccb(Assembler::above, above); // CF=0 & ZF=0
1759 } else {
1760 __ jccb(Assembler::above, done);
1761 }
1762 __ jccb(Assembler::parity, nan); // PF=1
1763 __ jccb(Assembler::below, below); // CF=1
1764
1765 // equal
1766 // Using bitwise operations is a low cost way to compute the correct result
1767 // for zero and non-zero inputs in this scenario except for NaN, which is
1768 // handled separately. The mantissa and exponent are valid with either
1769 // bitwise operation. For zero inputs, the sign bit is chosen according to
1770 // whether a minimum or maximum value is required.
1771 if (min) {
1772 // Negative sign preserved when available (e.g., min(+0, -0) -> -0)
1773 __ vpor(dst, a, b, Assembler::AVX_128bit);
1774 } else {
1775 // Positive sign preserved when available (e.g., max(+0, -0) -> +0)
1776 __ vpand(dst, a, b, Assembler::AVX_128bit);
1777 }
1778 __ jmp(done);
1779
1780 __ bind(above);
1781 movfp(masm, pt, dst, min ? b : a, rt);
1782 __ jmp(done);
1783
1784 __ bind(nan);
1785 if (pt == fp_prec_hlf) {
1786 __ movl(rt, 0x00007e00); // Float16.NaN
1787 __ evmovw(dst, rt);
1788 } else if (pt == fp_prec_flt) {
1789 __ movl(rt, 0x7fc00000); // Float.NaN
1790 __ movdl(dst, rt);
1791 } else {
1792 __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
1793 __ movdq(dst, rt);
1794 }
1795 __ jmp(done);
1796
1797 __ bind(below);
1798 movfp(masm, pt, dst, min ? a : b, rt);
1799
1800 __ bind(done);
1801 }
1802
1803 //=============================================================================
1804 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1805
1806 int ConstantTable::calculate_table_base_offset() const {
1807 return 0; // absolute addressing, no offset
1808 }
1809
1810 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1811 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1812 ShouldNotReachHere();
1813 }
1814
1815 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1816 // Empty encoding
1817 }
1818
1819 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1820 return 0;
1821 }
1822
1823 #ifndef PRODUCT
1824 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1825 st->print("# MachConstantBaseNode (empty encoding)");
1826 }
1827 #endif
1828
1829
1830 //=============================================================================
1831 #ifndef PRODUCT
1832 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1833 Compile* C = ra_->C;
1834
1835 int framesize = C->output()->frame_size_in_bytes();
1836 int bangsize = C->output()->bang_size_in_bytes();
1837 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1838 // Remove wordSize for return addr which is already pushed.
1839 framesize -= wordSize;
1840
1841 if (C->output()->need_stack_bang(bangsize)) {
1842 framesize -= wordSize;
1843 st->print("# stack bang (%d bytes)", bangsize);
1844 st->print("\n\t");
1845 st->print("pushq rbp\t# Save rbp");
1846 if (PreserveFramePointer) {
1847 st->print("\n\t");
1848 st->print("movq rbp, rsp\t# Save the caller's SP into rbp");
1849 }
1850 if (framesize) {
1851 st->print("\n\t");
1852 st->print("subq rsp, #%d\t# Create frame",framesize);
1853 }
1854 } else {
1855 st->print("subq rsp, #%d\t# Create frame",framesize);
1856 st->print("\n\t");
1857 framesize -= wordSize;
1858 st->print("movq [rsp + #%d], rbp\t# Save rbp",framesize);
1859 if (PreserveFramePointer) {
1860 st->print("\n\t");
1861 st->print("movq rbp, rsp\t# Save the caller's SP into rbp");
1862 if (framesize > 0) {
1863 st->print("\n\t");
1864 st->print("addq rbp, #%d", framesize);
1865 }
1866 }
1867 }
1868
1869 if (VerifyStackAtCalls) {
1870 st->print("\n\t");
1871 framesize -= wordSize;
1872 st->print("movq [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
1873 #ifdef ASSERT
1874 st->print("\n\t");
1875 st->print("# stack alignment check");
1876 #endif
1877 }
1878 if (C->stub_function() != nullptr) {
1879 st->print("\n\t");
1880 st->print("cmpl [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
1881 st->print("\n\t");
1882 st->print("je fast_entry\t");
1883 st->print("\n\t");
1884 st->print("call #nmethod_entry_barrier_stub\t");
1885 st->print("\n\tfast_entry:");
1886 }
1887 st->cr();
1888 }
1889 #endif
1890
1891 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1892 Compile* C = ra_->C;
1893
1894 __ verified_entry(C);
1895
1896 if (ra_->C->stub_function() == nullptr) {
1897 __ entry_barrier();
1898 }
1899
1900 if (!Compile::current()->output()->in_scratch_emit_size()) {
1901 __ bind(*_verified_entry);
1902 }
1903
1904 C->output()->set_frame_complete(__ offset());
1905
1906 if (C->has_mach_constant_base_node()) {
1907 // NOTE: We set the table base offset here because users might be
1908 // emitted before MachConstantBaseNode.
1909 ConstantTable& constant_table = C->output()->constant_table();
1910 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1911 }
1912 }
1913
1914
1915 int MachPrologNode::reloc() const
1916 {
1917 return 0; // a large enough number
1918 }
1919
1920 //=============================================================================
1921 #ifndef PRODUCT
1922 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1923 {
1924 Compile* C = ra_->C;
1925 if (generate_vzeroupper(C)) {
1926 st->print("vzeroupper");
1927 st->cr(); st->print("\t");
1928 }
1929
1930 int framesize = C->output()->frame_size_in_bytes();
1931 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1932 // Remove word for return adr already pushed
1933 // and RBP
1934 framesize -= 2*wordSize;
1935
1936 if (framesize) {
1937 st->print_cr("addq rsp, %d\t# Destroy frame", framesize);
1938 st->print("\t");
1939 }
1940
1941 st->print_cr("popq rbp");
1942 if (do_polling() && C->is_method_compilation()) {
1943 st->print("\t");
1944 st->print_cr("cmpq rsp, poll_offset[r15_thread] \n\t"
1945 "ja #safepoint_stub\t"
1946 "# Safepoint: poll for GC");
1947 }
1948 }
1949 #endif
1950
1951 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
1952 {
1953 Compile* C = ra_->C;
1954
1955 if (generate_vzeroupper(C)) {
1956 // Clear upper bits of YMM registers when current compiled code uses
1957 // wide vectors to avoid AVX <-> SSE transition penalty during call.
1958 __ vzeroupper();
1959 }
1960
1961 // Subtract two words to account for return address and rbp
1962 int initial_framesize = C->output()->frame_size_in_bytes() - 2*wordSize;
1963 __ remove_frame(initial_framesize, C->needs_stack_repair());
1964
1965 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1966 __ reserved_stack_check();
1967 }
1968
1969 if (do_polling() && C->is_method_compilation()) {
1970 Label dummy_label;
1971 Label* code_stub = &dummy_label;
1972 if (!C->output()->in_scratch_emit_size()) {
1973 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1974 C->output()->add_stub(stub);
1975 code_stub = &stub->entry();
1976 }
1977 __ relocate(relocInfo::poll_return_type);
1978 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1979 }
1980 }
1981
1982 int MachEpilogNode::reloc() const
1983 {
1984 return 2; // a large enough number
1985 }
1986
1987 const Pipeline* MachEpilogNode::pipeline() const
1988 {
1989 return MachNode::pipeline_class();
1990 }
1991
1992 //=============================================================================
1993
1994 enum RC {
1995 rc_bad,
1996 rc_int,
1997 rc_kreg,
1998 rc_float,
1999 rc_stack
2000 };
2001
2002 static enum RC rc_class(OptoReg::Name reg)
2003 {
2004 if( !OptoReg::is_valid(reg) ) return rc_bad;
2005
2006 if (OptoReg::is_stack(reg)) return rc_stack;
2007
2008 VMReg r = OptoReg::as_VMReg(reg);
2009
2010 if (r->is_Register()) return rc_int;
2011
2012 if (r->is_KRegister()) return rc_kreg;
2013
2014 assert(r->is_XMMRegister(), "must be");
2015 return rc_float;
2016 }
2017
2018 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
2019 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
2020 int src_hi, int dst_hi, uint ireg, outputStream* st);
2021
2022 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
2023 int stack_offset, int reg, uint ireg, outputStream* st);
2024
2025 static void vec_stack_to_stack_helper(C2_MacroAssembler *masm, int src_offset,
2026 int dst_offset, uint ireg, outputStream* st) {
2027 if (masm) {
2028 switch (ireg) {
2029 case Op_VecS:
2030 __ movq(Address(rsp, -8), rax);
2031 __ movl(rax, Address(rsp, src_offset));
2032 __ movl(Address(rsp, dst_offset), rax);
2033 __ movq(rax, Address(rsp, -8));
2034 break;
2035 case Op_VecD:
2036 __ pushq(Address(rsp, src_offset));
2037 __ popq (Address(rsp, dst_offset));
2038 break;
2039 case Op_VecX:
2040 __ pushq(Address(rsp, src_offset));
2041 __ popq (Address(rsp, dst_offset));
2042 __ pushq(Address(rsp, src_offset+8));
2043 __ popq (Address(rsp, dst_offset+8));
2044 break;
2045 case Op_VecY:
2046 __ vmovdqu(Address(rsp, -32), xmm0);
2047 __ vmovdqu(xmm0, Address(rsp, src_offset));
2048 __ vmovdqu(Address(rsp, dst_offset), xmm0);
2049 __ vmovdqu(xmm0, Address(rsp, -32));
2050 break;
2051 case Op_VecZ:
2052 __ evmovdquq(Address(rsp, -64), xmm0, 2);
2053 __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
2054 __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
2055 __ evmovdquq(xmm0, Address(rsp, -64), 2);
2056 break;
2057 default:
2058 ShouldNotReachHere();
2059 }
2060 #ifndef PRODUCT
2061 } else {
2062 switch (ireg) {
2063 case Op_VecS:
2064 st->print("movq [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
2065 "movl rax, [rsp + #%d]\n\t"
2066 "movl [rsp + #%d], rax\n\t"
2067 "movq rax, [rsp - #8]",
2068 src_offset, dst_offset);
2069 break;
2070 case Op_VecD:
2071 st->print("pushq [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
2072 "popq [rsp + #%d]",
2073 src_offset, dst_offset);
2074 break;
2075 case Op_VecX:
2076 st->print("pushq [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
2077 "popq [rsp + #%d]\n\t"
2078 "pushq [rsp + #%d]\n\t"
2079 "popq [rsp + #%d]",
2080 src_offset, dst_offset, src_offset+8, dst_offset+8);
2081 break;
2082 case Op_VecY:
2083 st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
2084 "vmovdqu xmm0, [rsp + #%d]\n\t"
2085 "vmovdqu [rsp + #%d], xmm0\n\t"
2086 "vmovdqu xmm0, [rsp - #32]",
2087 src_offset, dst_offset);
2088 break;
2089 case Op_VecZ:
2090 st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
2091 "vmovdqu xmm0, [rsp + #%d]\n\t"
2092 "vmovdqu [rsp + #%d], xmm0\n\t"
2093 "vmovdqu xmm0, [rsp - #64]",
2094 src_offset, dst_offset);
2095 break;
2096 default:
2097 ShouldNotReachHere();
2098 }
2099 #endif
2100 }
2101 }
2102
2103 uint MachSpillCopyNode::implementation(C2_MacroAssembler* masm,
2104 PhaseRegAlloc* ra_,
2105 bool do_size,
2106 outputStream* st) const {
2107 assert(masm != nullptr || st != nullptr, "sanity");
2108 // Get registers to move
2109 OptoReg::Name src_second = ra_->get_reg_second(in(1));
2110 OptoReg::Name src_first = ra_->get_reg_first(in(1));
2111 OptoReg::Name dst_second = ra_->get_reg_second(this);
2112 OptoReg::Name dst_first = ra_->get_reg_first(this);
2113
2114 enum RC src_first_rc = rc_class(src_first);
2115 enum RC dst_first_rc = rc_class(dst_first);
2116
2117 assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
2118 "must move at least 1 register" );
2119
2120 if (src_first == dst_first && src_second == dst_second) {
2121 // Self copy, no move
2122 return 0;
2123 }
2124 if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_pvectmask() == nullptr) {
2125 uint ireg = ideal_reg();
2126 assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
2127 assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
2128 if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
2129 // mem -> mem
2130 int src_offset = ra_->reg2offset(src_first);
2131 int dst_offset = ra_->reg2offset(dst_first);
2132 vec_stack_to_stack_helper(masm, src_offset, dst_offset, ireg, st);
2133 } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
2134 vec_mov_helper(masm, src_first, dst_first, src_second, dst_second, ireg, st);
2135 } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
2136 int stack_offset = ra_->reg2offset(dst_first);
2137 vec_spill_helper(masm, false, stack_offset, src_first, ireg, st);
2138 } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
2139 int stack_offset = ra_->reg2offset(src_first);
2140 vec_spill_helper(masm, true, stack_offset, dst_first, ireg, st);
2141 } else {
2142 ShouldNotReachHere();
2143 }
2144 return 0;
2145 }
2146 if (src_first_rc == rc_stack) {
2147 // mem ->
2148 if (dst_first_rc == rc_stack) {
2149 // mem -> mem
2150 assert(src_second != dst_first, "overlap");
2151 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2152 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2153 // 64-bit
2154 int src_offset = ra_->reg2offset(src_first);
2155 int dst_offset = ra_->reg2offset(dst_first);
2156 if (masm) {
2157 __ pushq(Address(rsp, src_offset));
2158 __ popq (Address(rsp, dst_offset));
2159 #ifndef PRODUCT
2160 } else {
2161 st->print("pushq [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
2162 "popq [rsp + #%d]",
2163 src_offset, dst_offset);
2164 #endif
2165 }
2166 } else {
2167 // 32-bit
2168 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2169 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2170 // No pushl/popl, so:
2171 int src_offset = ra_->reg2offset(src_first);
2172 int dst_offset = ra_->reg2offset(dst_first);
2173 if (masm) {
2174 __ movq(Address(rsp, -8), rax);
2175 __ movl(rax, Address(rsp, src_offset));
2176 __ movl(Address(rsp, dst_offset), rax);
2177 __ movq(rax, Address(rsp, -8));
2178 #ifndef PRODUCT
2179 } else {
2180 st->print("movq [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
2181 "movl rax, [rsp + #%d]\n\t"
2182 "movl [rsp + #%d], rax\n\t"
2183 "movq rax, [rsp - #8]",
2184 src_offset, dst_offset);
2185 #endif
2186 }
2187 }
2188 return 0;
2189 } else if (dst_first_rc == rc_int) {
2190 // mem -> gpr
2191 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2192 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2193 // 64-bit
2194 int offset = ra_->reg2offset(src_first);
2195 if (masm) {
2196 __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2197 #ifndef PRODUCT
2198 } else {
2199 st->print("movq %s, [rsp + #%d]\t# spill",
2200 Matcher::regName[dst_first],
2201 offset);
2202 #endif
2203 }
2204 } else {
2205 // 32-bit
2206 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2207 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2208 int offset = ra_->reg2offset(src_first);
2209 if (masm) {
2210 __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2211 #ifndef PRODUCT
2212 } else {
2213 st->print("movl %s, [rsp + #%d]\t# spill",
2214 Matcher::regName[dst_first],
2215 offset);
2216 #endif
2217 }
2218 }
2219 return 0;
2220 } else if (dst_first_rc == rc_float) {
2221 // mem-> xmm
2222 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2223 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2224 // 64-bit
2225 int offset = ra_->reg2offset(src_first);
2226 if (masm) {
2227 __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2228 #ifndef PRODUCT
2229 } else {
2230 st->print("%s %s, [rsp + #%d]\t# spill",
2231 UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
2232 Matcher::regName[dst_first],
2233 offset);
2234 #endif
2235 }
2236 } else {
2237 // 32-bit
2238 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2239 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2240 int offset = ra_->reg2offset(src_first);
2241 if (masm) {
2242 __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2243 #ifndef PRODUCT
2244 } else {
2245 st->print("movss %s, [rsp + #%d]\t# spill",
2246 Matcher::regName[dst_first],
2247 offset);
2248 #endif
2249 }
2250 }
2251 return 0;
2252 } else if (dst_first_rc == rc_kreg) {
2253 // mem -> kreg
2254 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2255 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2256 // 64-bit
2257 int offset = ra_->reg2offset(src_first);
2258 if (masm) {
2259 __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2260 #ifndef PRODUCT
2261 } else {
2262 st->print("kmovq %s, [rsp + #%d]\t# spill",
2263 Matcher::regName[dst_first],
2264 offset);
2265 #endif
2266 }
2267 }
2268 return 0;
2269 }
2270 } else if (src_first_rc == rc_int) {
2271 // gpr ->
2272 if (dst_first_rc == rc_stack) {
2273 // gpr -> mem
2274 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2275 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2276 // 64-bit
2277 int offset = ra_->reg2offset(dst_first);
2278 if (masm) {
2279 __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
2280 #ifndef PRODUCT
2281 } else {
2282 st->print("movq [rsp + #%d], %s\t# spill",
2283 offset,
2284 Matcher::regName[src_first]);
2285 #endif
2286 }
2287 } else {
2288 // 32-bit
2289 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2290 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2291 int offset = ra_->reg2offset(dst_first);
2292 if (masm) {
2293 __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
2294 #ifndef PRODUCT
2295 } else {
2296 st->print("movl [rsp + #%d], %s\t# spill",
2297 offset,
2298 Matcher::regName[src_first]);
2299 #endif
2300 }
2301 }
2302 return 0;
2303 } else if (dst_first_rc == rc_int) {
2304 // gpr -> gpr
2305 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2306 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2307 // 64-bit
2308 if (masm) {
2309 __ movq(as_Register(Matcher::_regEncode[dst_first]),
2310 as_Register(Matcher::_regEncode[src_first]));
2311 #ifndef PRODUCT
2312 } else {
2313 st->print("movq %s, %s\t# spill",
2314 Matcher::regName[dst_first],
2315 Matcher::regName[src_first]);
2316 #endif
2317 }
2318 return 0;
2319 } else {
2320 // 32-bit
2321 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2322 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2323 if (masm) {
2324 __ movl(as_Register(Matcher::_regEncode[dst_first]),
2325 as_Register(Matcher::_regEncode[src_first]));
2326 #ifndef PRODUCT
2327 } else {
2328 st->print("movl %s, %s\t# spill",
2329 Matcher::regName[dst_first],
2330 Matcher::regName[src_first]);
2331 #endif
2332 }
2333 return 0;
2334 }
2335 } else if (dst_first_rc == rc_float) {
2336 // gpr -> xmm
2337 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2338 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2339 // 64-bit
2340 if (masm) {
2341 __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
2342 #ifndef PRODUCT
2343 } else {
2344 st->print("movdq %s, %s\t# spill",
2345 Matcher::regName[dst_first],
2346 Matcher::regName[src_first]);
2347 #endif
2348 }
2349 } else {
2350 // 32-bit
2351 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2352 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2353 if (masm) {
2354 __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
2355 #ifndef PRODUCT
2356 } else {
2357 st->print("movdl %s, %s\t# spill",
2358 Matcher::regName[dst_first],
2359 Matcher::regName[src_first]);
2360 #endif
2361 }
2362 }
2363 return 0;
2364 } else if (dst_first_rc == rc_kreg) {
2365 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2366 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2367 // 64-bit
2368 if (masm) {
2369 __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
2370 #ifndef PRODUCT
2371 } else {
2372 st->print("kmovq %s, %s\t# spill",
2373 Matcher::regName[dst_first],
2374 Matcher::regName[src_first]);
2375 #endif
2376 }
2377 }
2378 Unimplemented();
2379 return 0;
2380 }
2381 } else if (src_first_rc == rc_float) {
2382 // xmm ->
2383 if (dst_first_rc == rc_stack) {
2384 // xmm -> mem
2385 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2386 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2387 // 64-bit
2388 int offset = ra_->reg2offset(dst_first);
2389 if (masm) {
2390 __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
2391 #ifndef PRODUCT
2392 } else {
2393 st->print("movsd [rsp + #%d], %s\t# spill",
2394 offset,
2395 Matcher::regName[src_first]);
2396 #endif
2397 }
2398 } else {
2399 // 32-bit
2400 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2401 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2402 int offset = ra_->reg2offset(dst_first);
2403 if (masm) {
2404 __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
2405 #ifndef PRODUCT
2406 } else {
2407 st->print("movss [rsp + #%d], %s\t# spill",
2408 offset,
2409 Matcher::regName[src_first]);
2410 #endif
2411 }
2412 }
2413 return 0;
2414 } else if (dst_first_rc == rc_int) {
2415 // xmm -> gpr
2416 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2417 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2418 // 64-bit
2419 if (masm) {
2420 __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2421 #ifndef PRODUCT
2422 } else {
2423 st->print("movdq %s, %s\t# spill",
2424 Matcher::regName[dst_first],
2425 Matcher::regName[src_first]);
2426 #endif
2427 }
2428 } else {
2429 // 32-bit
2430 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2431 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2432 if (masm) {
2433 __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2434 #ifndef PRODUCT
2435 } else {
2436 st->print("movdl %s, %s\t# spill",
2437 Matcher::regName[dst_first],
2438 Matcher::regName[src_first]);
2439 #endif
2440 }
2441 }
2442 return 0;
2443 } else if (dst_first_rc == rc_float) {
2444 // xmm -> xmm
2445 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2446 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2447 // 64-bit
2448 if (masm) {
2449 __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2450 #ifndef PRODUCT
2451 } else {
2452 st->print("%s %s, %s\t# spill",
2453 UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
2454 Matcher::regName[dst_first],
2455 Matcher::regName[src_first]);
2456 #endif
2457 }
2458 } else {
2459 // 32-bit
2460 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2461 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2462 if (masm) {
2463 __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2464 #ifndef PRODUCT
2465 } else {
2466 st->print("%s %s, %s\t# spill",
2467 UseXmmRegToRegMoveAll ? "movaps" : "movss ",
2468 Matcher::regName[dst_first],
2469 Matcher::regName[src_first]);
2470 #endif
2471 }
2472 }
2473 return 0;
2474 } else if (dst_first_rc == rc_kreg) {
2475 assert(false, "Illegal spilling");
2476 return 0;
2477 }
2478 } else if (src_first_rc == rc_kreg) {
2479 if (dst_first_rc == rc_stack) {
2480 // mem -> kreg
2481 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2482 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2483 // 64-bit
2484 int offset = ra_->reg2offset(dst_first);
2485 if (masm) {
2486 __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
2487 #ifndef PRODUCT
2488 } else {
2489 st->print("kmovq [rsp + #%d] , %s\t# spill",
2490 offset,
2491 Matcher::regName[src_first]);
2492 #endif
2493 }
2494 }
2495 return 0;
2496 } else if (dst_first_rc == rc_int) {
2497 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2498 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2499 // 64-bit
2500 if (masm) {
2501 __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
2502 #ifndef PRODUCT
2503 } else {
2504 st->print("kmovq %s, %s\t# spill",
2505 Matcher::regName[dst_first],
2506 Matcher::regName[src_first]);
2507 #endif
2508 }
2509 }
2510 Unimplemented();
2511 return 0;
2512 } else if (dst_first_rc == rc_kreg) {
2513 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2514 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2515 // 64-bit
2516 if (masm) {
2517 __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
2518 #ifndef PRODUCT
2519 } else {
2520 st->print("kmovq %s, %s\t# spill",
2521 Matcher::regName[dst_first],
2522 Matcher::regName[src_first]);
2523 #endif
2524 }
2525 }
2526 return 0;
2527 } else if (dst_first_rc == rc_float) {
2528 assert(false, "Illegal spill");
2529 return 0;
2530 }
2531 }
2532
2533 assert(0," foo ");
2534 Unimplemented();
2535 return 0;
2536 }
2537
2538 #ifndef PRODUCT
2539 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
2540 implementation(nullptr, ra_, false, st);
2541 }
2542 #endif
2543
2544 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2545 implementation(masm, ra_, false, nullptr);
2546 }
2547
2548 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2549 return MachNode::size(ra_);
2550 }
2551
2552 //=============================================================================
2553 #ifndef PRODUCT
2554 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2555 {
2556 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2557 int reg = ra_->get_reg_first(this);
2558 st->print("leaq %s, [rsp + #%d]\t# box lock",
2559 Matcher::regName[reg], offset);
2560 }
2561 #endif
2562
2563 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2564 {
2565 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2566 int reg = ra_->get_encode(this);
2567
2568 __ lea(as_Register(reg), Address(rsp, offset));
2569 }
2570
2571 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
2572 {
2573 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2574 if (ra_->get_encode(this) > 15) {
2575 return (offset < 0x80) ? 6 : 9; // REX2
2576 } else {
2577 return (offset < 0x80) ? 5 : 8; // REX
2578 }
2579 }
2580
2581 //=============================================================================
2582 #ifndef PRODUCT
2583 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2584 {
2585 st->print_cr("MachVEPNode");
2586 }
2587 #endif
2588
2589 void MachVEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2590 {
2591 CodeBuffer* cbuf = masm->code();
2592 if (!_verified) {
2593 __ ic_check(1);
2594 } else {
2595 if (ra_->C->stub_function() == nullptr) {
2596 // Emit the entry barrier in a temporary frame before unpacking because
2597 // it can deopt, which would require packing the scalarized args again.
2598 __ verified_entry(ra_->C, 0);
2599 __ entry_barrier();
2600 int initial_framesize = ra_->C->output()->frame_size_in_bytes() - 2*wordSize;
2601 __ remove_frame(initial_framesize, false);
2602 }
2603 // Unpack inline type args passed as oop and then jump to
2604 // the verified entry point (skipping the unverified entry).
2605 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2606 // Emit code for verified entry and save increment for stack repair on return
2607 __ verified_entry(ra_->C, sp_inc);
2608 if (Compile::current()->output()->in_scratch_emit_size()) {
2609 Label dummy_verified_entry;
2610 __ jmp(dummy_verified_entry);
2611 } else {
2612 __ jmp(*_verified_entry);
2613 }
2614 }
2615 if (ra_->C->stub_function() == nullptr) {
2616 // Pad so that the next call to MachVEPNode::emit() starts out with the
2617 // correct alignment. This is needed by entry_barrier() to align the
2618 // compare. But unfortunately we need to align all 4 MachVEPNodes because
2619 // entry point offsets are computed using scratch_emit_size(), so starting
2620 // alignment must match the alignment of the scratch buffer, otherwise the sizes
2621 // will be off.
2622 __ align(4);
2623 }
2624 }
2625
2626 //=============================================================================
2627 #ifndef PRODUCT
2628 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2629 {
2630 st->print_cr("movl rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2631 st->print_cr("\tcmpl rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
2632 st->print_cr("\tjne SharedRuntime::_ic_miss_stub");
2633 }
2634 #endif
2635
2636 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2637 {
2638 __ ic_check(InteriorEntryAlignment);
2639 }
2640
2641
2642 //=============================================================================
2643
2644 bool Matcher::supports_vector_calling_convention(void) {
2645 return EnableVectorSupport;
2646 }
2647
2648 static bool is_ndd_demotable_opr1(const MachNode* mdef) {
2649 return ((mdef->flags() & Node::PD::Flag_ndd_demotable_opr1) != 0);
2650 }
2651
2652 static bool is_ndd_demotable_opr2(const MachNode* mdef) {
2653 return ((mdef->flags() & Node::PD::Flag_ndd_demotable_opr2) != 0);
2654 }
2655
2656 #ifdef ASSERT
2657 static bool is_ndd_demotable(const MachNode* mdef) {
2658 return (is_ndd_demotable_opr1(mdef) || is_ndd_demotable_opr2(mdef));
2659 }
2660 #endif
2661
2662 bool Matcher::is_register_biasing_candidate(const MachNode* mdef,
2663 int oper_index) {
2664 if (mdef == nullptr) {
2665 return false;
2666 }
2667
2668 if (mdef->num_opnds() <= oper_index || mdef->operand_index(oper_index) < 0 ||
2669 mdef->in(mdef->operand_index(oper_index)) == nullptr) {
2670 assert(oper_index != 1 || !is_ndd_demotable_opr1(mdef), "%s", mdef->Name());
2671 assert(oper_index != 2 || !is_ndd_demotable_opr2(mdef), "%s", mdef->Name());
2672 return false;
2673 }
2674
2675 // Complex memory operand covers multiple incoming edges needed for
2676 // address computation. Biasing def towards any address component will not
2677 // result in NDD demotion by assembler.
2678 if (mdef->operand_num_edges(oper_index) != 1) {
2679 return false;
2680 }
2681
2682 // Demotion candidate must be register mask compatible with definition.
2683 const RegMask& oper_mask = mdef->in_RegMask(mdef->operand_index(oper_index));
2684 if (!oper_mask.overlap(mdef->out_RegMask())) {
2685 assert(!is_ndd_demotable(mdef), "%s", mdef->Name());
2686 return false;
2687 }
2688
2689 switch (oper_index) {
2690 // First operand of MachNode corresponding to Intel APX NDD selection
2691 // pattern can share its assigned register with definition operand if
2692 // their live ranges do not overlap. In such a scenario we can demote
2693 // it to legacy map0/map1 instruction by replacing its 4-byte extended
2694 // EVEX prefix with shorter REX/REX2 encoding. Demotion candidates
2695 // are decorated with a special flag by instruction selector.
2696 case 1:
2697 return is_ndd_demotable_opr1(mdef);
2698
2699 // Definition operand of commutative operation can be biased towards second
2700 // operand.
2701 case 2:
2702 return is_ndd_demotable_opr2(mdef);
2703
2704 // Current scheme only selects up to two biasing candidates
2705 default:
2706 assert(false, "unhandled operand index: %s", mdef->Name());
2707 break;
2708 }
2709
2710 return false;
2711 }
2712
2713 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2714 assert(EnableVectorSupport, "sanity");
2715 int lo = XMM0_num;
2716 int hi = XMM0b_num;
2717 if (ideal_reg == Op_VecX) hi = XMM0d_num;
2718 else if (ideal_reg == Op_VecY) hi = XMM0h_num;
2719 else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
2720 return OptoRegPair(hi, lo);
2721 }
2722
2723 // Is this branch offset short enough that a short branch can be used?
2724 //
2725 // NOTE: If the platform does not provide any short branch variants, then
2726 // this method should return false for offset 0.
2727 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2728 // The passed offset is relative to address of the branch.
2729 // On 86 a branch displacement is calculated relative to address
2730 // of a next instruction.
2731 offset -= br_size;
2732
2733 // the short version of jmpConUCF2 contains multiple branches,
2734 // making the reach slightly less
2735 if (rule == jmpConUCF2_rule)
2736 return (-126 <= offset && offset <= 125);
2737 return (-128 <= offset && offset <= 127);
2738 }
2739
2740 #ifdef ASSERT
2741 // Return whether or not this register is ever used as an argument.
2742 bool Matcher::can_be_java_arg(int reg)
2743 {
2744 return
2745 reg == RDI_num || reg == RDI_H_num ||
2746 reg == RSI_num || reg == RSI_H_num ||
2747 reg == RDX_num || reg == RDX_H_num ||
2748 reg == RCX_num || reg == RCX_H_num ||
2749 reg == R8_num || reg == R8_H_num ||
2750 reg == R9_num || reg == R9_H_num ||
2751 reg == R12_num || reg == R12_H_num ||
2752 reg == XMM0_num || reg == XMM0b_num ||
2753 reg == XMM1_num || reg == XMM1b_num ||
2754 reg == XMM2_num || reg == XMM2b_num ||
2755 reg == XMM3_num || reg == XMM3b_num ||
2756 reg == XMM4_num || reg == XMM4b_num ||
2757 reg == XMM5_num || reg == XMM5b_num ||
2758 reg == XMM6_num || reg == XMM6b_num ||
2759 reg == XMM7_num || reg == XMM7b_num;
2760 }
2761 #endif
2762
2763 uint Matcher::int_pressure_limit()
2764 {
2765 return (INTPRESSURE == -1) ? _INT_REG_mask.size() : INTPRESSURE;
2766 }
2767
2768 uint Matcher::float_pressure_limit()
2769 {
2770 // After experiment around with different values, the following default threshold
2771 // works best for LCM's register pressure scheduling on x64.
2772 uint dec_count = VM_Version::supports_evex() ? 4 : 2;
2773 uint default_float_pressure_threshold = _FLOAT_REG_mask.size() - dec_count;
2774 return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
2775 }
2776
2777 // Register for the first projection of an int pair
2778 const RegMask& Matcher::firstI_proj_mask() {
2779 return INT_RAX_REG_mask();
2780 }
2781
2782 // Register for the second projection of an int pair
2783 const RegMask& Matcher::secondI_proj_mask() {
2784 return INT_RDX_REG_mask();
2785 }
2786
2787 // Register for the first projection of a long pair
2788 const RegMask& Matcher::firstL_proj_mask() {
2789 return LONG_RAX_REG_mask();
2790 }
2791
2792 // Register for the second projection of a long pair
2793 const RegMask& Matcher::secondL_proj_mask() {
2794 return LONG_RDX_REG_mask();
2795 }
2796
2797 %}
2798
2799 source_hpp %{
2800 // Header information of the source block.
2801 // Method declarations/definitions which are used outside
2802 // the ad-scope can conveniently be defined here.
2803 //
2804 // To keep related declarations/definitions/uses close together,
2805 // we switch between source %{ }% and source_hpp %{ }% freely as needed.
2806
2807 #include "runtime/vm_version.hpp"
2808
2809 class NativeJump;
2810
2811 class CallStubImpl {
2812
2813 //--------------------------------------------------------------
2814 //---< Used for optimization in Compile::shorten_branches >---
2815 //--------------------------------------------------------------
2816
2817 public:
2818 // Size of call trampoline stub.
2819 static uint size_call_trampoline() {
2820 return 0; // no call trampolines on this platform
2821 }
2822
2823 // number of relocations needed by a call trampoline stub
2824 static uint reloc_call_trampoline() {
2825 return 0; // no call trampolines on this platform
2826 }
2827 };
2828
2829 class HandlerImpl {
2830
2831 public:
2832
2833 static int emit_deopt_handler(C2_MacroAssembler* masm);
2834
2835 static uint size_deopt_handler() {
2836 // one call and one jmp.
2837 return 7;
2838 }
2839 };
2840
2841 inline Assembler::AvxVectorLen vector_length_encoding(int bytes) {
2842 switch(bytes) {
2843 case 4: // fall-through
2844 case 8: // fall-through
2845 case 16: return Assembler::AVX_128bit;
2846 case 32: return Assembler::AVX_256bit;
2847 case 64: return Assembler::AVX_512bit;
2848
2849 default: {
2850 ShouldNotReachHere();
2851 return Assembler::AVX_NoVec;
2852 }
2853 }
2854 }
2855
2856 static inline Assembler::AvxVectorLen vector_length_encoding(const Node* n) {
2857 return vector_length_encoding(Matcher::vector_length_in_bytes(n));
2858 }
2859
2860 static inline Assembler::AvxVectorLen vector_length_encoding(const MachNode* use, MachOper* opnd) {
2861 uint def_idx = use->operand_index(opnd);
2862 Node* def = use->in(def_idx);
2863 return vector_length_encoding(def);
2864 }
2865
2866 static inline bool is_vector_popcount_predicate(BasicType bt) {
2867 return (is_subword_type(bt) && VM_Version::supports_avx512_bitalg()) ||
2868 (is_non_subword_integral_type(bt) && VM_Version::supports_avx512_vpopcntdq());
2869 }
2870
2871 static inline bool is_clz_non_subword_predicate_evex(BasicType bt, int vlen_bytes) {
2872 return is_non_subword_integral_type(bt) && VM_Version::supports_avx512cd() &&
2873 (VM_Version::supports_avx512vl() || vlen_bytes == 64);
2874 }
2875
2876 class Node::PD {
2877 public:
2878 enum NodeFlags : uint64_t {
2879 Flag_intel_jcc_erratum = Node::_last_flag << 1,
2880 Flag_sets_carry_flag = Node::_last_flag << 2,
2881 Flag_sets_parity_flag = Node::_last_flag << 3,
2882 Flag_sets_zero_flag = Node::_last_flag << 4,
2883 Flag_sets_overflow_flag = Node::_last_flag << 5,
2884 Flag_sets_sign_flag = Node::_last_flag << 6,
2885 Flag_clears_carry_flag = Node::_last_flag << 7,
2886 Flag_clears_parity_flag = Node::_last_flag << 8,
2887 Flag_clears_zero_flag = Node::_last_flag << 9,
2888 Flag_clears_overflow_flag = Node::_last_flag << 10,
2889 Flag_clears_sign_flag = Node::_last_flag << 11,
2890 Flag_ndd_demotable_opr1 = Node::_last_flag << 12,
2891 Flag_ndd_demotable_opr2 = Node::_last_flag << 13,
2892 _last_flag = Flag_ndd_demotable_opr2
2893 };
2894 };
2895
2896 %} // end source_hpp
2897
2898 source %{
2899
2900 #include "opto/addnode.hpp"
2901 #include "c2_intelJccErratum_x86.hpp"
2902
2903 void PhaseOutput::pd_perform_mach_node_analysis() {
2904 if (VM_Version::has_intel_jcc_erratum()) {
2905 int extra_padding = IntelJccErratum::tag_affected_machnodes(C, C->cfg(), C->regalloc());
2906 _buf_sizes._code += extra_padding;
2907 }
2908 }
2909
2910 int MachNode::pd_alignment_required() const {
2911 if (VM_Version::has_intel_jcc_erratum() && IntelJccErratum::is_jcc_erratum_branch(this)) {
2912 // Conservatively add worst case padding. We assume that relocInfo::addr_unit() is 1 on x86.
2913 return IntelJccErratum::largest_jcc_size() + 1;
2914 } else {
2915 return 1;
2916 }
2917 }
2918
2919 int MachNode::compute_padding(int current_offset) const {
2920 if (flags() & Node::PD::Flag_intel_jcc_erratum) {
2921 Compile* C = Compile::current();
2922 PhaseOutput* output = C->output();
2923 Block* block = output->block();
2924 int index = output->index();
2925 return IntelJccErratum::compute_padding(current_offset, this, block, index, C->regalloc());
2926 } else {
2927 return 0;
2928 }
2929 }
2930
2931 // Emit deopt handler code.
2932 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) {
2933
2934 // Note that the code buffer's insts_mark is always relative to insts.
2935 // That's why we must use the macroassembler to generate a handler.
2936 address base = __ start_a_stub(size_deopt_handler());
2937 if (base == nullptr) {
2938 ciEnv::current()->record_failure("CodeCache is full");
2939 return 0; // CodeBuffer::expand failed
2940 }
2941 int offset = __ offset();
2942
2943 Label start;
2944 __ bind(start);
2945
2946 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2947
2948 int entry_offset = __ offset();
2949
2950 __ jmp(start);
2951
2952 assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow %d", (__ offset() - offset));
2953 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2954 "out of bounds read in post-call NOP check");
2955 __ end_a_stub();
2956 return entry_offset;
2957 }
2958
2959 static Assembler::Width widthForType(BasicType bt) {
2960 if (bt == T_BYTE) {
2961 return Assembler::B;
2962 } else if (bt == T_SHORT) {
2963 return Assembler::W;
2964 } else if (bt == T_INT) {
2965 return Assembler::D;
2966 } else {
2967 assert(bt == T_LONG, "not a long: %s", type2name(bt));
2968 return Assembler::Q;
2969 }
2970 }
2971
2972 //=============================================================================
2973
2974 // Float masks come from different places depending on platform.
2975 static address float_signmask() { return StubRoutines::x86::float_sign_mask(); }
2976 static address float_signflip() { return StubRoutines::x86::float_sign_flip(); }
2977 static address double_signmask() { return StubRoutines::x86::double_sign_mask(); }
2978 static address double_signflip() { return StubRoutines::x86::double_sign_flip(); }
2979 static address vector_short_to_byte_mask() { return StubRoutines::x86::vector_short_to_byte_mask(); }
2980 static address vector_int_to_byte_mask() { return StubRoutines::x86::vector_int_to_byte_mask(); }
2981 static address vector_byte_perm_mask() { return StubRoutines::x86::vector_byte_perm_mask(); }
2982 static address vector_long_sign_mask() { return StubRoutines::x86::vector_long_sign_mask(); }
2983 static address vector_all_bits_set() { return StubRoutines::x86::vector_all_bits_set(); }
2984 static address vector_int_mask_cmp_bits() { return StubRoutines::x86::vector_int_mask_cmp_bits(); }
2985 static address vector_int_to_short_mask() { return StubRoutines::x86::vector_int_to_short_mask(); }
2986 static address vector_byte_shufflemask() { return StubRoutines::x86::vector_byte_shuffle_mask(); }
2987 static address vector_short_shufflemask() { return StubRoutines::x86::vector_short_shuffle_mask(); }
2988 static address vector_int_shufflemask() { return StubRoutines::x86::vector_int_shuffle_mask(); }
2989 static address vector_long_shufflemask() { return StubRoutines::x86::vector_long_shuffle_mask(); }
2990 static address vector_32_bit_mask() { return StubRoutines::x86::vector_32_bit_mask(); }
2991 static address vector_64_bit_mask() { return StubRoutines::x86::vector_64_bit_mask(); }
2992 static address vector_float_signflip() { return StubRoutines::x86::vector_float_sign_flip();}
2993 static address vector_double_signflip() { return StubRoutines::x86::vector_double_sign_flip();}
2994
2995 //=============================================================================
2996 bool Matcher::match_rule_supported(int opcode) {
2997 if (!has_match_rule(opcode)) {
2998 return false; // no match rule present
2999 }
3000 switch (opcode) {
3001 case Op_AbsVL:
3002 case Op_StoreVectorScatter:
3003 if (UseAVX < 3) {
3004 return false;
3005 }
3006 break;
3007 case Op_PopCountI:
3008 case Op_PopCountL:
3009 if (!UsePopCountInstruction) {
3010 return false;
3011 }
3012 break;
3013 case Op_PopCountVI:
3014 if (UseAVX < 2) {
3015 return false;
3016 }
3017 break;
3018 case Op_CompressV:
3019 case Op_ExpandV:
3020 case Op_PopCountVL:
3021 if (UseAVX < 2) {
3022 return false;
3023 }
3024 break;
3025 case Op_MulVI:
3026 if ((UseSSE < 4) && (UseAVX < 1)) { // only with SSE4_1 or AVX
3027 return false;
3028 }
3029 break;
3030 case Op_MulVL:
3031 if (UseSSE < 4) { // only with SSE4_1 or AVX
3032 return false;
3033 }
3034 break;
3035 case Op_MulReductionVL:
3036 if (VM_Version::supports_avx512dq() == false) {
3037 return false;
3038 }
3039 break;
3040 case Op_AbsVB:
3041 case Op_AbsVS:
3042 case Op_AbsVI:
3043 case Op_AddReductionVI:
3044 case Op_AndReductionV:
3045 case Op_OrReductionV:
3046 case Op_XorReductionV:
3047 if (UseSSE < 3) { // requires at least SSSE3
3048 return false;
3049 }
3050 break;
3051 case Op_MaxHF:
3052 case Op_MinHF:
3053 if (!VM_Version::supports_avx512vlbw()) {
3054 return false;
3055 } // fallthrough
3056 case Op_AddHF:
3057 case Op_DivHF:
3058 case Op_FmaHF:
3059 case Op_MulHF:
3060 case Op_ReinterpretS2HF:
3061 case Op_ReinterpretHF2S:
3062 case Op_SubHF:
3063 case Op_SqrtHF:
3064 if (!VM_Version::supports_avx512_fp16()) {
3065 return false;
3066 }
3067 break;
3068 case Op_VectorLoadShuffle:
3069 case Op_VectorRearrange:
3070 case Op_MulReductionVI:
3071 if (UseSSE < 4) { // requires at least SSE4
3072 return false;
3073 }
3074 break;
3075 case Op_IsInfiniteF:
3076 case Op_IsInfiniteD:
3077 if (!VM_Version::supports_avx512dq()) {
3078 return false;
3079 }
3080 break;
3081 case Op_SqrtVD:
3082 case Op_SqrtVF:
3083 case Op_VectorMaskCmp:
3084 case Op_VectorCastB2X:
3085 case Op_VectorCastS2X:
3086 case Op_VectorCastI2X:
3087 case Op_VectorCastL2X:
3088 case Op_VectorCastF2X:
3089 case Op_VectorCastD2X:
3090 case Op_VectorUCastB2X:
3091 case Op_VectorUCastS2X:
3092 case Op_VectorUCastI2X:
3093 case Op_VectorMaskCast:
3094 if (UseAVX < 1) { // enabled for AVX only
3095 return false;
3096 }
3097 break;
3098 case Op_PopulateIndex:
3099 if (UseAVX < 2) {
3100 return false;
3101 }
3102 break;
3103 case Op_RoundVF:
3104 if (UseAVX < 2) { // enabled for AVX2 only
3105 return false;
3106 }
3107 break;
3108 case Op_RoundVD:
3109 if (UseAVX < 3) {
3110 return false; // enabled for AVX3 only
3111 }
3112 break;
3113 case Op_CompareAndSwapL:
3114 case Op_CompareAndSwapP:
3115 break;
3116 case Op_StrIndexOf:
3117 if (!UseSSE42Intrinsics) {
3118 return false;
3119 }
3120 break;
3121 case Op_StrIndexOfChar:
3122 if (!UseSSE42Intrinsics) {
3123 return false;
3124 }
3125 break;
3126 case Op_OnSpinWait:
3127 if (VM_Version::supports_on_spin_wait() == false) {
3128 return false;
3129 }
3130 break;
3131 case Op_MulVB:
3132 case Op_LShiftVB:
3133 case Op_RShiftVB:
3134 case Op_URShiftVB:
3135 case Op_VectorInsert:
3136 case Op_VectorLoadMask:
3137 case Op_VectorStoreMask:
3138 case Op_VectorBlend:
3139 if (UseSSE < 4) {
3140 return false;
3141 }
3142 break;
3143 case Op_MaxD:
3144 case Op_MaxF:
3145 case Op_MinD:
3146 case Op_MinF:
3147 if (UseAVX < 1) { // enabled for AVX only
3148 return false;
3149 }
3150 break;
3151 case Op_CacheWB:
3152 case Op_CacheWBPreSync:
3153 case Op_CacheWBPostSync:
3154 if (!VM_Version::supports_data_cache_line_flush()) {
3155 return false;
3156 }
3157 break;
3158 case Op_ExtractB:
3159 case Op_ExtractL:
3160 case Op_ExtractI:
3161 case Op_RoundDoubleMode:
3162 if (UseSSE < 4) {
3163 return false;
3164 }
3165 break;
3166 case Op_RoundDoubleModeV:
3167 if (VM_Version::supports_avx() == false) {
3168 return false; // 128bit vroundpd is not available
3169 }
3170 break;
3171 case Op_LoadVectorGather:
3172 case Op_LoadVectorGatherMasked:
3173 if (UseAVX < 2) {
3174 return false;
3175 }
3176 break;
3177 case Op_FmaF:
3178 case Op_FmaD:
3179 case Op_FmaVD:
3180 case Op_FmaVF:
3181 if (!UseFMA) {
3182 return false;
3183 }
3184 break;
3185 case Op_MacroLogicV:
3186 if (UseAVX < 3 || !UseVectorMacroLogic) {
3187 return false;
3188 }
3189 break;
3190
3191 case Op_VectorCmpMasked:
3192 if (UseAVX < 3 || !UseCountTrailingZerosInstruction) {
3193 return false;
3194 }
3195 break;
3196 case Op_VectorMaskGen:
3197 if (UseAVX < 3 || !VM_Version::supports_bmi2()) {
3198 return false;
3199 }
3200 break;
3201 case Op_VectorMaskFirstTrue:
3202 case Op_VectorMaskLastTrue:
3203 case Op_VectorMaskTrueCount:
3204 case Op_VectorMaskToLong:
3205 if (UseAVX < 1) {
3206 return false;
3207 }
3208 break;
3209 case Op_RoundF:
3210 case Op_RoundD:
3211 break;
3212 case Op_CopySignD:
3213 case Op_CopySignF:
3214 if (UseAVX < 3) {
3215 return false;
3216 }
3217 if (!VM_Version::supports_avx512vl()) {
3218 return false;
3219 }
3220 break;
3221 case Op_CompressBits:
3222 case Op_ExpandBits:
3223 if (!VM_Version::supports_bmi2()) {
3224 return false;
3225 }
3226 break;
3227 case Op_CompressM:
3228 if (!VM_Version::supports_avx512vl() || !VM_Version::supports_bmi2()) {
3229 return false;
3230 }
3231 break;
3232 case Op_ConvF2HF:
3233 case Op_ConvHF2F:
3234 if (!VM_Version::supports_float16()) {
3235 return false;
3236 }
3237 break;
3238 case Op_VectorCastF2HF:
3239 case Op_VectorCastHF2F:
3240 if (!VM_Version::supports_f16c() && !VM_Version::supports_evex()) {
3241 return false;
3242 }
3243 break;
3244 }
3245 return true; // Match rules are supported by default.
3246 }
3247
3248 //------------------------------------------------------------------------
3249
3250 static inline bool is_pop_count_instr_target(BasicType bt) {
3251 return (is_subword_type(bt) && VM_Version::supports_avx512_bitalg()) ||
3252 (is_non_subword_integral_type(bt) && VM_Version::supports_avx512_vpopcntdq());
3253 }
3254
3255 bool Matcher::match_rule_supported_auto_vectorization(int opcode, int vlen, BasicType bt) {
3256 return match_rule_supported_vector(opcode, vlen, bt);
3257 }
3258
3259 // Identify extra cases that we might want to provide match rules for vector nodes and
3260 // other intrinsics guarded with vector length (vlen) and element type (bt).
3261 bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
3262 if (!match_rule_supported(opcode)) {
3263 return false;
3264 }
3265 // Matcher::vector_size_supported() restricts vector sizes in the following way (see Matcher::vector_width_in_bytes):
3266 // * SSE2 supports 128bit vectors for all types;
3267 // * AVX1 supports 256bit vectors only for FLOAT and DOUBLE types;
3268 // * AVX2 supports 256bit vectors for all types;
3269 // * AVX512F supports 512bit vectors only for INT, FLOAT, and DOUBLE types;
3270 // * AVX512BW supports 512bit vectors for BYTE, SHORT, and CHAR types.
3271 // There's also a limit on minimum vector size supported: 2 elements (or 4 bytes for BYTE).
3272 // And MaxVectorSize is taken into account as well.
3273 if (!vector_size_supported(bt, vlen)) {
3274 return false;
3275 }
3276 // Special cases which require vector length follow:
3277 // * implementation limitations
3278 // * some 512bit vector operations on FLOAT and DOUBLE types require AVX512DQ
3279 // * 128bit vroundpd instruction is present only in AVX1
3280 int size_in_bits = vlen * type2aelembytes(bt) * BitsPerByte;
3281 switch (opcode) {
3282 case Op_MaxVHF:
3283 case Op_MinVHF:
3284 if (!VM_Version::supports_avx512bw()) {
3285 return false;
3286 }
3287 case Op_AddVHF:
3288 case Op_DivVHF:
3289 case Op_FmaVHF:
3290 case Op_MulVHF:
3291 case Op_SubVHF:
3292 case Op_SqrtVHF:
3293 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3294 return false;
3295 }
3296 if (!VM_Version::supports_avx512_fp16()) {
3297 return false;
3298 }
3299 break;
3300 case Op_AbsVF:
3301 case Op_NegVF:
3302 if ((vlen == 16) && (VM_Version::supports_avx512dq() == false)) {
3303 return false; // 512bit vandps and vxorps are not available
3304 }
3305 break;
3306 case Op_AbsVD:
3307 case Op_NegVD:
3308 if ((vlen == 8) && (VM_Version::supports_avx512dq() == false)) {
3309 return false; // 512bit vpmullq, vandpd and vxorpd are not available
3310 }
3311 break;
3312 case Op_RotateRightV:
3313 case Op_RotateLeftV:
3314 if (bt != T_INT && bt != T_LONG) {
3315 return false;
3316 } // fallthrough
3317 case Op_MacroLogicV:
3318 if (!VM_Version::supports_evex() ||
3319 ((size_in_bits != 512) && !VM_Version::supports_avx512vl())) {
3320 return false;
3321 }
3322 break;
3323 case Op_ClearArray:
3324 case Op_VectorMaskGen:
3325 case Op_VectorCmpMasked:
3326 if (!VM_Version::supports_avx512bw()) {
3327 return false;
3328 }
3329 if ((size_in_bits != 512) && !VM_Version::supports_avx512vl()) {
3330 return false;
3331 }
3332 break;
3333 case Op_LoadVectorMasked:
3334 case Op_StoreVectorMasked:
3335 if (!VM_Version::supports_avx512bw() && (is_subword_type(bt) || UseAVX < 1)) {
3336 return false;
3337 }
3338 break;
3339 case Op_UMinV:
3340 case Op_UMaxV:
3341 if (UseAVX == 0) {
3342 return false;
3343 }
3344 break;
3345 case Op_UMinReductionV:
3346 case Op_UMaxReductionV:
3347 if (UseAVX == 0) {
3348 return false;
3349 }
3350 if (bt == T_LONG && !VM_Version::supports_avx512vl()) {
3351 return false;
3352 }
3353 if (UseAVX > 2 && size_in_bits == 512 && !VM_Version::supports_avx512vl()) {
3354 return false;
3355 }
3356 break;
3357 case Op_MaxV:
3358 case Op_MinV:
3359 if (UseSSE < 4 && is_integral_type(bt)) {
3360 return false;
3361 }
3362 if ((bt == T_FLOAT || bt == T_DOUBLE)) {
3363 // Float/Double intrinsics are enabled for AVX family currently.
3364 if (UseAVX == 0) {
3365 return false;
3366 }
3367 if (UseAVX > 2 && (!VM_Version::supports_avx512dq() && size_in_bits == 512)) { // 512 bit Float/Double intrinsics need AVX512DQ
3368 return false;
3369 }
3370 }
3371 break;
3372 case Op_CallLeafVector:
3373 if (size_in_bits == 512 && !VM_Version::supports_avx512vlbwdq()) {
3374 return false;
3375 }
3376 break;
3377 case Op_AddReductionVI:
3378 if (bt == T_INT && (UseSSE < 3 || !VM_Version::supports_ssse3())) {
3379 return false;
3380 }
3381 // fallthrough
3382 case Op_AndReductionV:
3383 case Op_OrReductionV:
3384 case Op_XorReductionV:
3385 if (is_subword_type(bt) && (UseSSE < 4)) {
3386 return false;
3387 }
3388 break;
3389 case Op_MinReductionV:
3390 case Op_MaxReductionV:
3391 if ((bt == T_INT || is_subword_type(bt)) && UseSSE < 4) {
3392 return false;
3393 } else if (bt == T_LONG && (UseAVX < 3 || !VM_Version::supports_avx512vlbwdq())) {
3394 return false;
3395 }
3396 // Float/Double intrinsics enabled for AVX family.
3397 if (UseAVX == 0 && (bt == T_FLOAT || bt == T_DOUBLE)) {
3398 return false;
3399 }
3400 if (UseAVX > 2 && (!VM_Version::supports_avx512dq() && size_in_bits == 512)) {
3401 return false;
3402 }
3403 break;
3404 case Op_VectorBlend:
3405 if (UseAVX == 0 && size_in_bits < 128) {
3406 return false;
3407 }
3408 break;
3409 case Op_VectorTest:
3410 if (UseSSE < 4) {
3411 return false; // Implementation limitation
3412 } else if (size_in_bits < 32) {
3413 return false; // Implementation limitation
3414 }
3415 break;
3416 case Op_VectorLoadShuffle:
3417 case Op_VectorRearrange:
3418 if(vlen == 2) {
3419 return false; // Implementation limitation due to how shuffle is loaded
3420 } else if (size_in_bits == 256 && UseAVX < 2) {
3421 return false; // Implementation limitation
3422 }
3423 break;
3424 case Op_VectorLoadMask:
3425 case Op_VectorMaskCast:
3426 if (size_in_bits == 256 && UseAVX < 2) {
3427 return false; // Implementation limitation
3428 }
3429 // fallthrough
3430 case Op_VectorStoreMask:
3431 if (vlen == 2) {
3432 return false; // Implementation limitation
3433 }
3434 break;
3435 case Op_PopulateIndex:
3436 if (size_in_bits > 256 && !VM_Version::supports_avx512bw()) {
3437 return false;
3438 }
3439 break;
3440 case Op_VectorCastB2X:
3441 case Op_VectorCastS2X:
3442 case Op_VectorCastI2X:
3443 if (bt != T_DOUBLE && size_in_bits == 256 && UseAVX < 2) {
3444 return false;
3445 }
3446 break;
3447 case Op_VectorCastL2X:
3448 if (is_integral_type(bt) && size_in_bits == 256 && UseAVX < 2) {
3449 return false;
3450 } else if (!is_integral_type(bt) && !VM_Version::supports_avx512dq()) {
3451 return false;
3452 }
3453 break;
3454 case Op_VectorCastF2X: {
3455 // As per JLS section 5.1.3 narrowing conversion to sub-word types
3456 // happen after intermediate conversion to integer and special handling
3457 // code needs AVX2 vpcmpeqd instruction for 256 bit vectors.
3458 int src_size_in_bits = type2aelembytes(T_FLOAT) * vlen * BitsPerByte;
3459 if (is_integral_type(bt) && src_size_in_bits == 256 && UseAVX < 2) {
3460 return false;
3461 }
3462 }
3463 // fallthrough
3464 case Op_VectorCastD2X:
3465 if (bt == T_LONG && !VM_Version::supports_avx512dq()) {
3466 return false;
3467 }
3468 break;
3469 case Op_VectorCastF2HF:
3470 case Op_VectorCastHF2F:
3471 if (!VM_Version::supports_f16c() &&
3472 ((!VM_Version::supports_evex() ||
3473 ((size_in_bits != 512) && !VM_Version::supports_avx512vl())))) {
3474 return false;
3475 }
3476 break;
3477 case Op_RoundVD:
3478 if (!VM_Version::supports_avx512dq()) {
3479 return false;
3480 }
3481 break;
3482 case Op_MulReductionVI:
3483 if (bt == T_BYTE && size_in_bits == 512 && !VM_Version::supports_avx512bw()) {
3484 return false;
3485 }
3486 break;
3487 case Op_LoadVectorGatherMasked:
3488 if (!is_subword_type(bt) && size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3489 return false;
3490 }
3491 if (is_subword_type(bt) &&
3492 ((size_in_bits > 256 && !VM_Version::supports_avx512bw()) ||
3493 (size_in_bits < 64) ||
3494 (bt == T_SHORT && !VM_Version::supports_bmi2()))) {
3495 return false;
3496 }
3497 break;
3498 case Op_StoreVectorScatterMasked:
3499 case Op_StoreVectorScatter:
3500 if (is_subword_type(bt)) {
3501 return false;
3502 } else if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3503 return false;
3504 }
3505 // fallthrough
3506 case Op_LoadVectorGather:
3507 if (!is_subword_type(bt) && size_in_bits == 64) {
3508 return false;
3509 }
3510 if (is_subword_type(bt) && size_in_bits < 64) {
3511 return false;
3512 }
3513 break;
3514 case Op_SaturatingAddV:
3515 case Op_SaturatingSubV:
3516 if (UseAVX < 1) {
3517 return false; // Implementation limitation
3518 }
3519 if (is_subword_type(bt) && size_in_bits == 512 && !VM_Version::supports_avx512bw()) {
3520 return false;
3521 }
3522 break;
3523 case Op_SelectFromTwoVector:
3524 if (size_in_bits < 128) {
3525 return false;
3526 }
3527 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3528 return false;
3529 }
3530 if (bt == T_SHORT && !VM_Version::supports_avx512bw()) {
3531 return false;
3532 }
3533 if (bt == T_BYTE && !VM_Version::supports_avx512_vbmi()) {
3534 return false;
3535 }
3536 if ((bt == T_INT || bt == T_FLOAT || bt == T_DOUBLE) && !VM_Version::supports_evex()) {
3537 return false;
3538 }
3539 break;
3540 case Op_MaskAll:
3541 if (!VM_Version::supports_evex()) {
3542 return false;
3543 }
3544 if ((vlen > 16 || is_subword_type(bt)) && !VM_Version::supports_avx512bw()) {
3545 return false;
3546 }
3547 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3548 return false;
3549 }
3550 break;
3551 case Op_VectorMaskCmp:
3552 if (vlen < 2 || size_in_bits < 32) {
3553 return false;
3554 }
3555 break;
3556 case Op_CompressM:
3557 if (UseAVX < 3 || !VM_Version::supports_bmi2()) {
3558 return false;
3559 }
3560 break;
3561 case Op_CompressV:
3562 case Op_ExpandV:
3563 if (is_subword_type(bt) && !VM_Version::supports_avx512_vbmi2()) {
3564 return false;
3565 }
3566 if (size_in_bits < 128 ) {
3567 return false;
3568 }
3569 case Op_VectorLongToMask:
3570 if (UseAVX < 1) {
3571 return false;
3572 }
3573 if (UseAVX < 3 && !VM_Version::supports_bmi2()) {
3574 return false;
3575 }
3576 break;
3577 case Op_SignumVD:
3578 case Op_SignumVF:
3579 if (UseAVX < 1) {
3580 return false;
3581 }
3582 break;
3583 case Op_PopCountVI:
3584 case Op_PopCountVL: {
3585 if (!is_pop_count_instr_target(bt) &&
3586 (size_in_bits == 512) && !VM_Version::supports_avx512bw()) {
3587 return false;
3588 }
3589 }
3590 break;
3591 case Op_ReverseV:
3592 case Op_ReverseBytesV:
3593 if (UseAVX < 2) {
3594 return false;
3595 }
3596 break;
3597 case Op_CountTrailingZerosV:
3598 case Op_CountLeadingZerosV:
3599 if (UseAVX < 2) {
3600 return false;
3601 }
3602 break;
3603 }
3604 return true; // Per default match rules are supported.
3605 }
3606
3607 bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
3608 // ADLC based match_rule_supported routine checks for the existence of pattern based
3609 // on IR opcode. Most of the unary/binary/ternary masked operation share the IR nodes
3610 // of their non-masked counterpart with mask edge being the differentiator.
3611 // This routine does a strict check on the existence of masked operation patterns
3612 // by returning a default false value for all the other opcodes apart from the
3613 // ones whose masked instruction patterns are defined in this file.
3614 if (!match_rule_supported_vector(opcode, vlen, bt)) {
3615 return false;
3616 }
3617
3618 int size_in_bits = vlen * type2aelembytes(bt) * BitsPerByte;
3619 if (size_in_bits != 512 && !VM_Version::supports_avx512vl()) {
3620 return false;
3621 }
3622 switch(opcode) {
3623 // Unary masked operations
3624 case Op_AbsVB:
3625 case Op_AbsVS:
3626 if(!VM_Version::supports_avx512bw()) {
3627 return false; // Implementation limitation
3628 }
3629 case Op_AbsVI:
3630 case Op_AbsVL:
3631 return true;
3632
3633 // Ternary masked operations
3634 case Op_FmaVF:
3635 case Op_FmaVD:
3636 return true;
3637
3638 case Op_MacroLogicV:
3639 if(bt != T_INT && bt != T_LONG) {
3640 return false;
3641 }
3642 return true;
3643
3644 // Binary masked operations
3645 case Op_AddVB:
3646 case Op_AddVS:
3647 case Op_SubVB:
3648 case Op_SubVS:
3649 case Op_MulVS:
3650 case Op_LShiftVS:
3651 case Op_RShiftVS:
3652 case Op_URShiftVS:
3653 assert(size_in_bits == 512 || VM_Version::supports_avx512vl(), "");
3654 if (!VM_Version::supports_avx512bw()) {
3655 return false; // Implementation limitation
3656 }
3657 return true;
3658
3659 case Op_MulVL:
3660 assert(size_in_bits == 512 || VM_Version::supports_avx512vl(), "");
3661 if (!VM_Version::supports_avx512dq()) {
3662 return false; // Implementation limitation
3663 }
3664 return true;
3665
3666 case Op_AndV:
3667 case Op_OrV:
3668 case Op_XorV:
3669 case Op_RotateRightV:
3670 case Op_RotateLeftV:
3671 if (bt != T_INT && bt != T_LONG) {
3672 return false; // Implementation limitation
3673 }
3674 return true;
3675
3676 case Op_VectorLoadMask:
3677 assert(size_in_bits == 512 || VM_Version::supports_avx512vl(), "");
3678 if (is_subword_type(bt) && !VM_Version::supports_avx512bw()) {
3679 return false;
3680 }
3681 return true;
3682
3683 case Op_AddVI:
3684 case Op_AddVL:
3685 case Op_AddVF:
3686 case Op_AddVD:
3687 case Op_SubVI:
3688 case Op_SubVL:
3689 case Op_SubVF:
3690 case Op_SubVD:
3691 case Op_MulVI:
3692 case Op_MulVF:
3693 case Op_MulVD:
3694 case Op_DivVF:
3695 case Op_DivVD:
3696 case Op_SqrtVF:
3697 case Op_SqrtVD:
3698 case Op_LShiftVI:
3699 case Op_LShiftVL:
3700 case Op_RShiftVI:
3701 case Op_RShiftVL:
3702 case Op_URShiftVI:
3703 case Op_URShiftVL:
3704 case Op_LoadVectorMasked:
3705 case Op_StoreVectorMasked:
3706 case Op_LoadVectorGatherMasked:
3707 case Op_StoreVectorScatterMasked:
3708 return true;
3709
3710 case Op_UMinV:
3711 case Op_UMaxV:
3712 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3713 return false;
3714 } // fallthrough
3715 case Op_MaxV:
3716 case Op_MinV:
3717 if (is_subword_type(bt) && !VM_Version::supports_avx512bw()) {
3718 return false; // Implementation limitation
3719 }
3720 if (is_floating_point_type(bt) && !VM_Version::supports_avx10_2()) {
3721 return false; // Implementation limitation
3722 }
3723 return true;
3724 case Op_SaturatingAddV:
3725 case Op_SaturatingSubV:
3726 if (!is_subword_type(bt)) {
3727 return false;
3728 }
3729 if (size_in_bits < 128 || !VM_Version::supports_avx512bw()) {
3730 return false; // Implementation limitation
3731 }
3732 return true;
3733
3734 case Op_VectorMaskCmp:
3735 if (is_subword_type(bt) && !VM_Version::supports_avx512bw()) {
3736 return false; // Implementation limitation
3737 }
3738 return true;
3739
3740 case Op_VectorRearrange:
3741 if (bt == T_SHORT && !VM_Version::supports_avx512bw()) {
3742 return false; // Implementation limitation
3743 }
3744 if (bt == T_BYTE && !VM_Version::supports_avx512_vbmi()) {
3745 return false; // Implementation limitation
3746 } else if ((bt == T_INT || bt == T_FLOAT) && size_in_bits < 256) {
3747 return false; // Implementation limitation
3748 }
3749 return true;
3750
3751 // Binary Logical operations
3752 case Op_AndVMask:
3753 case Op_OrVMask:
3754 case Op_XorVMask:
3755 if (vlen > 16 && !VM_Version::supports_avx512bw()) {
3756 return false; // Implementation limitation
3757 }
3758 return true;
3759
3760 case Op_PopCountVI:
3761 case Op_PopCountVL:
3762 if (!is_pop_count_instr_target(bt)) {
3763 return false;
3764 }
3765 return true;
3766
3767 case Op_MaskAll:
3768 return true;
3769
3770 case Op_CountLeadingZerosV:
3771 if (is_non_subword_integral_type(bt) && VM_Version::supports_avx512cd()) {
3772 return true;
3773 }
3774 default:
3775 return false;
3776 }
3777 }
3778
3779 bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* vt) {
3780 return false;
3781 }
3782
3783 // Return true if Vector::rearrange needs preparation of the shuffle argument
3784 bool Matcher::vector_rearrange_requires_load_shuffle(BasicType elem_bt, int vlen) {
3785 switch (elem_bt) {
3786 case T_BYTE: return false;
3787 case T_SHORT: return !VM_Version::supports_avx512bw();
3788 case T_INT: return !VM_Version::supports_avx();
3789 case T_LONG: return vlen < 8 && !VM_Version::supports_avx512vl();
3790 default:
3791 ShouldNotReachHere();
3792 return false;
3793 }
3794 }
3795
3796 bool Matcher::mask_op_prefers_predicate(int opcode, const TypeVect* vt) {
3797 // Prefer predicate if the mask type is "TypePVectMask".
3798 return vt->isa_pvectmask() != nullptr;
3799 }
3800
3801 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
3802 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
3803 bool legacy = (generic_opnd->opcode() == LEGVEC);
3804 if (!VM_Version::supports_avx512vlbwdq() && // KNL
3805 is_temp && !legacy && (ideal_reg == Op_VecZ)) {
3806 // Conservatively specialize 512bit vec TEMP operands to legVecZ (zmm0-15) on KNL.
3807 return new legVecZOper();
3808 }
3809 if (legacy) {
3810 switch (ideal_reg) {
3811 case Op_VecS: return new legVecSOper();
3812 case Op_VecD: return new legVecDOper();
3813 case Op_VecX: return new legVecXOper();
3814 case Op_VecY: return new legVecYOper();
3815 case Op_VecZ: return new legVecZOper();
3816 }
3817 } else {
3818 switch (ideal_reg) {
3819 case Op_VecS: return new vecSOper();
3820 case Op_VecD: return new vecDOper();
3821 case Op_VecX: return new vecXOper();
3822 case Op_VecY: return new vecYOper();
3823 case Op_VecZ: return new vecZOper();
3824 }
3825 }
3826 ShouldNotReachHere();
3827 return nullptr;
3828 }
3829
3830 bool Matcher::is_reg2reg_move(MachNode* m) {
3831 switch (m->rule()) {
3832 case MoveVec2Leg_rule:
3833 case MoveLeg2Vec_rule:
3834 case MoveF2VL_rule:
3835 case MoveF2LEG_rule:
3836 case MoveVL2F_rule:
3837 case MoveLEG2F_rule:
3838 case MoveD2VL_rule:
3839 case MoveD2LEG_rule:
3840 case MoveVL2D_rule:
3841 case MoveLEG2D_rule:
3842 return true;
3843 default:
3844 return false;
3845 }
3846 }
3847
3848 bool Matcher::is_generic_vector(MachOper* opnd) {
3849 switch (opnd->opcode()) {
3850 case VEC:
3851 case LEGVEC:
3852 return true;
3853 default:
3854 return false;
3855 }
3856 }
3857
3858 //------------------------------------------------------------------------
3859
3860 const RegMask* Matcher::predicate_reg_mask(void) {
3861 return &_VECTMASK_REG_mask;
3862 }
3863
3864 // Max vector size in bytes. 0 if not supported.
3865 int Matcher::vector_width_in_bytes(BasicType bt) {
3866 assert(is_java_primitive(bt), "only primitive type vectors");
3867 // SSE2 supports 128bit vectors for all types.
3868 // AVX2 supports 256bit vectors for all types.
3869 // AVX2/EVEX supports 512bit vectors for all types.
3870 int size = (UseAVX > 1) ? (1 << UseAVX) * 8 : 16;
3871 // AVX1 supports 256bit vectors only for FLOAT and DOUBLE.
3872 if (UseAVX > 0 && (bt == T_FLOAT || bt == T_DOUBLE))
3873 size = (UseAVX > 2) ? 64 : 32;
3874 if (UseAVX > 2 && (bt == T_BYTE || bt == T_SHORT || bt == T_CHAR))
3875 size = (VM_Version::supports_avx512bw()) ? 64 : 32;
3876 // Use flag to limit vector size.
3877 size = MIN2(size,(int)MaxVectorSize);
3878 // Minimum 2 values in vector (or 4 for bytes).
3879 switch (bt) {
3880 case T_DOUBLE:
3881 case T_LONG:
3882 if (size < 16) return 0;
3883 break;
3884 case T_FLOAT:
3885 case T_INT:
3886 if (size < 8) return 0;
3887 break;
3888 case T_BOOLEAN:
3889 if (size < 4) return 0;
3890 break;
3891 case T_CHAR:
3892 if (size < 4) return 0;
3893 break;
3894 case T_BYTE:
3895 if (size < 4) return 0;
3896 break;
3897 case T_SHORT:
3898 if (size < 4) return 0;
3899 break;
3900 default:
3901 ShouldNotReachHere();
3902 }
3903 return size;
3904 }
3905
3906 // Limits on vector size (number of elements) loaded into vector.
3907 int Matcher::max_vector_size(const BasicType bt) {
3908 return vector_width_in_bytes(bt)/type2aelembytes(bt);
3909 }
3910 int Matcher::min_vector_size(const BasicType bt) {
3911 int max_size = max_vector_size(bt);
3912 // Min size which can be loaded into vector is 4 bytes.
3913 int size = (type2aelembytes(bt) == 1) ? 4 : 2;
3914 // Support for calling svml double64 vectors
3915 if (bt == T_DOUBLE) {
3916 size = 1;
3917 }
3918 return MIN2(size,max_size);
3919 }
3920
3921 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
3922 // Limit the max vector size for auto vectorization to 256 bits (32 bytes)
3923 // by default on Cascade Lake
3924 if (VM_Version::is_default_intel_cascade_lake()) {
3925 return MIN2(Matcher::max_vector_size(bt), 32 / type2aelembytes(bt));
3926 }
3927 return Matcher::max_vector_size(bt);
3928 }
3929
3930 int Matcher::scalable_vector_reg_size(const BasicType bt) {
3931 return -1;
3932 }
3933
3934 // Vector ideal reg corresponding to specified size in bytes
3935 uint Matcher::vector_ideal_reg(int size) {
3936 assert(MaxVectorSize >= size, "");
3937 switch(size) {
3938 case 4: return Op_VecS;
3939 case 8: return Op_VecD;
3940 case 16: return Op_VecX;
3941 case 32: return Op_VecY;
3942 case 64: return Op_VecZ;
3943 }
3944 ShouldNotReachHere();
3945 return 0;
3946 }
3947
3948 // Check for shift by small constant as well
3949 static bool clone_shift(Node* shift, Matcher* matcher, Matcher::MStack& mstack, VectorSet& address_visited) {
3950 if (shift->Opcode() == Op_LShiftX && shift->in(2)->is_Con() &&
3951 shift->in(2)->get_int() <= 3 &&
3952 // Are there other uses besides address expressions?
3953 !matcher->is_visited(shift)) {
3954 address_visited.set(shift->_idx); // Flag as address_visited
3955 mstack.push(shift->in(2), Matcher::Visit);
3956 Node *conv = shift->in(1);
3957 // Allow Matcher to match the rule which bypass
3958 // ConvI2L operation for an array index on LP64
3959 // if the index value is positive.
3960 if (conv->Opcode() == Op_ConvI2L &&
3961 conv->as_Type()->type()->is_long()->_lo >= 0 &&
3962 // Are there other uses besides address expressions?
3963 !matcher->is_visited(conv)) {
3964 address_visited.set(conv->_idx); // Flag as address_visited
3965 mstack.push(conv->in(1), Matcher::Pre_Visit);
3966 } else {
3967 mstack.push(conv, Matcher::Pre_Visit);
3968 }
3969 return true;
3970 }
3971 return false;
3972 }
3973
3974 // This function identifies sub-graphs in which a 'load' node is
3975 // input to two different nodes, and such that it can be matched
3976 // with BMI instructions like blsi, blsr, etc.
3977 // Example : for b = -a[i] & a[i] can be matched to blsi r32, m32.
3978 // The graph is (AndL (SubL Con0 LoadL*) LoadL*), where LoadL*
3979 // refers to the same node.
3980 //
3981 // Match the generic fused operations pattern (op1 (op2 Con{ConType} mop) mop)
3982 // This is a temporary solution until we make DAGs expressible in ADL.
3983 template<typename ConType>
3984 class FusedPatternMatcher {
3985 Node* _op1_node;
3986 Node* _mop_node;
3987 int _con_op;
3988
3989 static int match_next(Node* n, int next_op, int next_op_idx) {
3990 if (n->in(1) == nullptr || n->in(2) == nullptr) {
3991 return -1;
3992 }
3993
3994 if (next_op_idx == -1) { // n is commutative, try rotations
3995 if (n->in(1)->Opcode() == next_op) {
3996 return 1;
3997 } else if (n->in(2)->Opcode() == next_op) {
3998 return 2;
3999 }
4000 } else {
4001 assert(next_op_idx > 0 && next_op_idx <= 2, "Bad argument index");
4002 if (n->in(next_op_idx)->Opcode() == next_op) {
4003 return next_op_idx;
4004 }
4005 }
4006 return -1;
4007 }
4008
4009 public:
4010 FusedPatternMatcher(Node* op1_node, Node* mop_node, int con_op) :
4011 _op1_node(op1_node), _mop_node(mop_node), _con_op(con_op) { }
4012
4013 bool match(int op1, int op1_op2_idx, // op1 and the index of the op1->op2 edge, -1 if op1 is commutative
4014 int op2, int op2_con_idx, // op2 and the index of the op2->con edge, -1 if op2 is commutative
4015 typename ConType::NativeType con_value) {
4016 if (_op1_node->Opcode() != op1) {
4017 return false;
4018 }
4019 if (_mop_node->outcnt() > 2) {
4020 return false;
4021 }
4022 op1_op2_idx = match_next(_op1_node, op2, op1_op2_idx);
4023 if (op1_op2_idx == -1) {
4024 return false;
4025 }
4026 // Memory operation must be the other edge
4027 int op1_mop_idx = (op1_op2_idx & 1) + 1;
4028
4029 // Check that the mop node is really what we want
4030 if (_op1_node->in(op1_mop_idx) == _mop_node) {
4031 Node* op2_node = _op1_node->in(op1_op2_idx);
4032 if (op2_node->outcnt() > 1) {
4033 return false;
4034 }
4035 assert(op2_node->Opcode() == op2, "Should be");
4036 op2_con_idx = match_next(op2_node, _con_op, op2_con_idx);
4037 if (op2_con_idx == -1) {
4038 return false;
4039 }
4040 // Memory operation must be the other edge
4041 int op2_mop_idx = (op2_con_idx & 1) + 1;
4042 // Check that the memory operation is the same node
4043 if (op2_node->in(op2_mop_idx) == _mop_node) {
4044 // Now check the constant
4045 const Type* con_type = op2_node->in(op2_con_idx)->bottom_type();
4046 if (con_type != Type::TOP && ConType::as_self(con_type)->get_con() == con_value) {
4047 return true;
4048 }
4049 }
4050 }
4051 return false;
4052 }
4053 };
4054
4055 static bool is_bmi_pattern(Node* n, Node* m) {
4056 assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "sanity");
4057 if (n != nullptr && m != nullptr) {
4058 if (m->Opcode() == Op_LoadI) {
4059 FusedPatternMatcher<TypeInt> bmii(n, m, Op_ConI);
4060 return bmii.match(Op_AndI, -1, Op_SubI, 1, 0) ||
4061 bmii.match(Op_AndI, -1, Op_AddI, -1, -1) ||
4062 bmii.match(Op_XorI, -1, Op_AddI, -1, -1);
4063 } else if (m->Opcode() == Op_LoadL) {
4064 FusedPatternMatcher<TypeLong> bmil(n, m, Op_ConL);
4065 return bmil.match(Op_AndL, -1, Op_SubL, 1, 0) ||
4066 bmil.match(Op_AndL, -1, Op_AddL, -1, -1) ||
4067 bmil.match(Op_XorL, -1, Op_AddL, -1, -1);
4068 }
4069 }
4070 return false;
4071 }
4072
4073 // Should the matcher clone input 'm' of node 'n'?
4074 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
4075 // If 'n' and 'm' are part of a graph for BMI instruction, clone the input 'm'.
4076 if (VM_Version::supports_bmi1() && VM_Version::supports_avx() && is_bmi_pattern(n, m)) {
4077 mstack.push(m, Visit);
4078 return true;
4079 }
4080 if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
4081 mstack.push(m, Visit); // m = ShiftCntV
4082 return true;
4083 }
4084 if (is_encode_and_store_pattern(n, m)) {
4085 mstack.push(m, Visit);
4086 return true;
4087 }
4088 return false;
4089 }
4090
4091 // Should the Matcher clone shifts on addressing modes, expecting them
4092 // to be subsumed into complex addressing expressions or compute them
4093 // into registers?
4094 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
4095 Node *off = m->in(AddPNode::Offset);
4096 if (off->is_Con()) {
4097 address_visited.test_set(m->_idx); // Flag as address_visited
4098 Node *adr = m->in(AddPNode::Address);
4099
4100 // Intel can handle 2 adds in addressing mode, with one of them using an immediate offset.
4101 // AtomicAdd is not an addressing expression.
4102 // Cheap to find it by looking for screwy base.
4103 if (adr->is_AddP() &&
4104 !adr->in(AddPNode::Base)->is_top() &&
4105 !adr->in(AddPNode::Offset)->is_Con() &&
4106 off->get_long() == (int) (off->get_long()) && // immL32
4107 // Are there other uses besides address expressions?
4108 !is_visited(adr)) {
4109 address_visited.set(adr->_idx); // Flag as address_visited
4110 Node *shift = adr->in(AddPNode::Offset);
4111 if (!clone_shift(shift, this, mstack, address_visited)) {
4112 mstack.push(shift, Pre_Visit);
4113 }
4114 mstack.push(adr->in(AddPNode::Address), Pre_Visit);
4115 mstack.push(adr->in(AddPNode::Base), Pre_Visit);
4116 } else {
4117 mstack.push(adr, Pre_Visit);
4118 }
4119
4120 // Clone X+offset as it also folds into most addressing expressions
4121 mstack.push(off, Visit);
4122 mstack.push(m->in(AddPNode::Base), Pre_Visit);
4123 return true;
4124 } else if (clone_shift(off, this, mstack, address_visited)) {
4125 address_visited.test_set(m->_idx); // Flag as address_visited
4126 mstack.push(m->in(AddPNode::Address), Pre_Visit);
4127 mstack.push(m->in(AddPNode::Base), Pre_Visit);
4128 return true;
4129 }
4130 return false;
4131 }
4132
4133 static inline Assembler::ComparisonPredicate booltest_pred_to_comparison_pred(int bt) {
4134 switch (bt) {
4135 case BoolTest::eq:
4136 return Assembler::eq;
4137 case BoolTest::ne:
4138 return Assembler::neq;
4139 case BoolTest::le:
4140 case BoolTest::ule:
4141 return Assembler::le;
4142 case BoolTest::ge:
4143 case BoolTest::uge:
4144 return Assembler::nlt;
4145 case BoolTest::lt:
4146 case BoolTest::ult:
4147 return Assembler::lt;
4148 case BoolTest::gt:
4149 case BoolTest::ugt:
4150 return Assembler::nle;
4151 default : ShouldNotReachHere(); return Assembler::_false;
4152 }
4153 }
4154
4155 static inline Assembler::ComparisonPredicateFP booltest_pred_to_comparison_pred_fp(int bt) {
4156 switch (bt) {
4157 case BoolTest::eq: return Assembler::EQ_OQ; // ordered non-signaling
4158 // As per JLS 15.21.1, != of NaNs is true. Thus use unordered compare.
4159 case BoolTest::ne: return Assembler::NEQ_UQ; // unordered non-signaling
4160 case BoolTest::le: return Assembler::LE_OQ; // ordered non-signaling
4161 case BoolTest::ge: return Assembler::GE_OQ; // ordered non-signaling
4162 case BoolTest::lt: return Assembler::LT_OQ; // ordered non-signaling
4163 case BoolTest::gt: return Assembler::GT_OQ; // ordered non-signaling
4164 default: ShouldNotReachHere(); return Assembler::FALSE_OS;
4165 }
4166 }
4167
4168 // Helper methods for MachSpillCopyNode::implementation().
4169 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
4170 int src_hi, int dst_hi, uint ireg, outputStream* st) {
4171 assert(ireg == Op_VecS || // 32bit vector
4172 ((src_lo & 1) == 0 && (src_lo + 1) == src_hi &&
4173 (dst_lo & 1) == 0 && (dst_lo + 1) == dst_hi),
4174 "no non-adjacent vector moves" );
4175 if (masm) {
4176 switch (ireg) {
4177 case Op_VecS: // copy whole register
4178 case Op_VecD:
4179 case Op_VecX:
4180 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4181 __ movdqu(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]));
4182 } else {
4183 __ vextractf32x4(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]), 0x0);
4184 }
4185 break;
4186 case Op_VecY:
4187 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4188 __ vmovdqu(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]));
4189 } else {
4190 __ vextractf64x4(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]), 0x0);
4191 }
4192 break;
4193 case Op_VecZ:
4194 __ evmovdquq(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]), 2);
4195 break;
4196 default:
4197 ShouldNotReachHere();
4198 }
4199 #ifndef PRODUCT
4200 } else {
4201 switch (ireg) {
4202 case Op_VecS:
4203 case Op_VecD:
4204 case Op_VecX:
4205 st->print("movdqu %s,%s\t# spill",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
4206 break;
4207 case Op_VecY:
4208 case Op_VecZ:
4209 st->print("vmovdqu %s,%s\t# spill",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
4210 break;
4211 default:
4212 ShouldNotReachHere();
4213 }
4214 #endif
4215 }
4216 }
4217
4218 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
4219 int stack_offset, int reg, uint ireg, outputStream* st) {
4220 if (masm) {
4221 if (is_load) {
4222 switch (ireg) {
4223 case Op_VecS:
4224 __ movdl(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4225 break;
4226 case Op_VecD:
4227 __ movq(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4228 break;
4229 case Op_VecX:
4230 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4231 __ movdqu(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4232 } else {
4233 __ vpxor(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), 2);
4234 __ vinsertf32x4(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset),0x0);
4235 }
4236 break;
4237 case Op_VecY:
4238 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4239 __ vmovdqu(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4240 } else {
4241 __ vpxor(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), 2);
4242 __ vinsertf64x4(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset),0x0);
4243 }
4244 break;
4245 case Op_VecZ:
4246 __ evmovdquq(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset), 2);
4247 break;
4248 default:
4249 ShouldNotReachHere();
4250 }
4251 } else { // store
4252 switch (ireg) {
4253 case Op_VecS:
4254 __ movdl(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4255 break;
4256 case Op_VecD:
4257 __ movq(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4258 break;
4259 case Op_VecX:
4260 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4261 __ movdqu(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4262 }
4263 else {
4264 __ vextractf32x4(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]), 0x0);
4265 }
4266 break;
4267 case Op_VecY:
4268 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4269 __ vmovdqu(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4270 }
4271 else {
4272 __ vextractf64x4(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]), 0x0);
4273 }
4274 break;
4275 case Op_VecZ:
4276 __ evmovdquq(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]), 2);
4277 break;
4278 default:
4279 ShouldNotReachHere();
4280 }
4281 }
4282 #ifndef PRODUCT
4283 } else {
4284 if (is_load) {
4285 switch (ireg) {
4286 case Op_VecS:
4287 st->print("movd %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4288 break;
4289 case Op_VecD:
4290 st->print("movq %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4291 break;
4292 case Op_VecX:
4293 st->print("movdqu %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4294 break;
4295 case Op_VecY:
4296 case Op_VecZ:
4297 st->print("vmovdqu %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4298 break;
4299 default:
4300 ShouldNotReachHere();
4301 }
4302 } else { // store
4303 switch (ireg) {
4304 case Op_VecS:
4305 st->print("movd [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4306 break;
4307 case Op_VecD:
4308 st->print("movq [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4309 break;
4310 case Op_VecX:
4311 st->print("movdqu [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4312 break;
4313 case Op_VecY:
4314 case Op_VecZ:
4315 st->print("vmovdqu [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4316 break;
4317 default:
4318 ShouldNotReachHere();
4319 }
4320 }
4321 #endif
4322 }
4323 }
4324
4325 template <class T>
4326 static inline GrowableArray<jbyte>* vreplicate_imm(BasicType bt, T con, int len) {
4327 int size = type2aelembytes(bt) * len;
4328 GrowableArray<jbyte>* val = new GrowableArray<jbyte>(size, size, 0);
4329 for (int i = 0; i < len; i++) {
4330 int offset = i * type2aelembytes(bt);
4331 switch (bt) {
4332 case T_BYTE: val->at(i) = con; break;
4333 case T_SHORT: {
4334 jshort c = con;
4335 memcpy(val->adr_at(offset), &c, sizeof(jshort));
4336 break;
4337 }
4338 case T_INT: {
4339 jint c = con;
4340 memcpy(val->adr_at(offset), &c, sizeof(jint));
4341 break;
4342 }
4343 case T_LONG: {
4344 jlong c = con;
4345 memcpy(val->adr_at(offset), &c, sizeof(jlong));
4346 break;
4347 }
4348 case T_FLOAT: {
4349 jfloat c = con;
4350 memcpy(val->adr_at(offset), &c, sizeof(jfloat));
4351 break;
4352 }
4353 case T_DOUBLE: {
4354 jdouble c = con;
4355 memcpy(val->adr_at(offset), &c, sizeof(jdouble));
4356 break;
4357 }
4358 default: assert(false, "%s", type2name(bt));
4359 }
4360 }
4361 return val;
4362 }
4363
4364 static inline jlong high_bit_set(BasicType bt) {
4365 switch (bt) {
4366 case T_BYTE: return 0x8080808080808080;
4367 case T_SHORT: return 0x8000800080008000;
4368 case T_INT: return 0x8000000080000000;
4369 case T_LONG: return 0x8000000000000000;
4370 default:
4371 ShouldNotReachHere();
4372 return 0;
4373 }
4374 }
4375
4376 #ifndef PRODUCT
4377 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
4378 st->print("nop \t# %d bytes pad for loops and calls", _count);
4379 }
4380 #endif
4381
4382 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
4383 __ nop(_count);
4384 }
4385
4386 uint MachNopNode::size(PhaseRegAlloc*) const {
4387 return _count;
4388 }
4389
4390 #ifndef PRODUCT
4391 void MachBreakpointNode::format(PhaseRegAlloc*, outputStream* st) const {
4392 st->print("# breakpoint");
4393 }
4394 #endif
4395
4396 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const {
4397 __ int3();
4398 }
4399
4400 uint MachBreakpointNode::size(PhaseRegAlloc* ra_) const {
4401 return MachNode::size(ra_);
4402 }
4403
4404 %}
4405
4406 //----------ENCODING BLOCK-----------------------------------------------------
4407 // This block specifies the encoding classes used by the compiler to
4408 // output byte streams. Encoding classes are parameterized macros
4409 // used by Machine Instruction Nodes in order to generate the bit
4410 // encoding of the instruction. Operands specify their base encoding
4411 // interface with the interface keyword. There are currently
4412 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
4413 // COND_INTER. REG_INTER causes an operand to generate a function
4414 // which returns its register number when queried. CONST_INTER causes
4415 // an operand to generate a function which returns the value of the
4416 // constant when queried. MEMORY_INTER causes an operand to generate
4417 // four functions which return the Base Register, the Index Register,
4418 // the Scale Value, and the Offset Value of the operand when queried.
4419 // COND_INTER causes an operand to generate six functions which return
4420 // the encoding code (ie - encoding bits for the instruction)
4421 // associated with each basic boolean condition for a conditional
4422 // instruction.
4423 //
4424 // Instructions specify two basic values for encoding. Again, a
4425 // function is available to check if the constant displacement is an
4426 // oop. They use the ins_encode keyword to specify their encoding
4427 // classes (which must be a sequence of enc_class names, and their
4428 // parameters, specified in the encoding block), and they use the
4429 // opcode keyword to specify, in order, their primary, secondary, and
4430 // tertiary opcode. Only the opcode sections which a particular
4431 // instruction needs for encoding need to be specified.
4432 encode %{
4433 enc_class cdql_enc(no_rax_rdx_RegI div)
4434 %{
4435 // Full implementation of Java idiv and irem; checks for
4436 // special case as described in JVM spec., p.243 & p.271.
4437 //
4438 // normal case special case
4439 //
4440 // input : rax: dividend min_int
4441 // reg: divisor -1
4442 //
4443 // output: rax: quotient (= rax idiv reg) min_int
4444 // rdx: remainder (= rax irem reg) 0
4445 //
4446 // Code sequnce:
4447 //
4448 // 0: 3d 00 00 00 80 cmp $0x80000000,%eax
4449 // 5: 75 07/08 jne e <normal>
4450 // 7: 33 d2 xor %edx,%edx
4451 // [div >= 8 -> offset + 1]
4452 // [REX_B]
4453 // 9: 83 f9 ff cmp $0xffffffffffffffff,$div
4454 // c: 74 03/04 je 11 <done>
4455 // 000000000000000e <normal>:
4456 // e: 99 cltd
4457 // [div >= 8 -> offset + 1]
4458 // [REX_B]
4459 // f: f7 f9 idiv $div
4460 // 0000000000000011 <done>:
4461 Label normal;
4462 Label done;
4463
4464 // cmp $0x80000000,%eax
4465 __ cmpl(as_Register(RAX_enc), 0x80000000);
4466
4467 // jne e <normal>
4468 __ jccb(Assembler::notEqual, normal);
4469
4470 // xor %edx,%edx
4471 __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
4472
4473 // cmp $0xffffffffffffffff,%ecx
4474 __ cmpl($div$$Register, -1);
4475
4476 // je 11 <done>
4477 __ jccb(Assembler::equal, done);
4478
4479 // <normal>
4480 // cltd
4481 __ bind(normal);
4482 __ cdql();
4483
4484 // idivl
4485 // <done>
4486 __ idivl($div$$Register);
4487 __ bind(done);
4488 %}
4489
4490 enc_class cdqq_enc(no_rax_rdx_RegL div)
4491 %{
4492 // Full implementation of Java ldiv and lrem; checks for
4493 // special case as described in JVM spec., p.243 & p.271.
4494 //
4495 // normal case special case
4496 //
4497 // input : rax: dividend min_long
4498 // reg: divisor -1
4499 //
4500 // output: rax: quotient (= rax idiv reg) min_long
4501 // rdx: remainder (= rax irem reg) 0
4502 //
4503 // Code sequnce:
4504 //
4505 // 0: 48 ba 00 00 00 00 00 mov $0x8000000000000000,%rdx
4506 // 7: 00 00 80
4507 // a: 48 39 d0 cmp %rdx,%rax
4508 // d: 75 08 jne 17 <normal>
4509 // f: 33 d2 xor %edx,%edx
4510 // 11: 48 83 f9 ff cmp $0xffffffffffffffff,$div
4511 // 15: 74 05 je 1c <done>
4512 // 0000000000000017 <normal>:
4513 // 17: 48 99 cqto
4514 // 19: 48 f7 f9 idiv $div
4515 // 000000000000001c <done>:
4516 Label normal;
4517 Label done;
4518
4519 // mov $0x8000000000000000,%rdx
4520 __ mov64(as_Register(RDX_enc), 0x8000000000000000);
4521
4522 // cmp %rdx,%rax
4523 __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
4524
4525 // jne 17 <normal>
4526 __ jccb(Assembler::notEqual, normal);
4527
4528 // xor %edx,%edx
4529 __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
4530
4531 // cmp $0xffffffffffffffff,$div
4532 __ cmpq($div$$Register, -1);
4533
4534 // je 1e <done>
4535 __ jccb(Assembler::equal, done);
4536
4537 // <normal>
4538 // cqto
4539 __ bind(normal);
4540 __ cdqq();
4541
4542 // idivq (note: must be emitted by the user of this rule)
4543 // <done>
4544 __ idivq($div$$Register);
4545 __ bind(done);
4546 %}
4547
4548 enc_class clear_avx %{
4549 DEBUG_ONLY(int off0 = __ offset());
4550 if (generate_vzeroupper(Compile::current())) {
4551 // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
4552 // Clear upper bits of YMM registers when current compiled code uses
4553 // wide vectors to avoid AVX <-> SSE transition penalty during call.
4554 __ vzeroupper();
4555 }
4556 DEBUG_ONLY(int off1 = __ offset());
4557 assert(off1 - off0 == clear_avx_size(), "correct size prediction");
4558 %}
4559
4560 enc_class Java_To_Runtime(method meth) %{
4561 __ lea(r10, RuntimeAddress((address)$meth$$method));
4562 __ call(r10);
4563 __ post_call_nop();
4564 %}
4565
4566 enc_class Java_Static_Call(method meth)
4567 %{
4568 // JAVA STATIC CALL
4569 // CALL to fixup routine. Fixup routine uses ScopeDesc info to
4570 // determine who we intended to call.
4571 if (!_method) {
4572 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
4573 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
4574 // The NOP here is purely to ensure that eliding a call to
4575 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
4576 __ nop(5);
4577 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
4578 } else {
4579 int method_index = resolved_method_index(masm);
4580 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
4581 : static_call_Relocation::spec(method_index);
4582 address mark = __ pc();
4583 int call_offset = __ offset();
4584 __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
4585 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
4586 // Calls of the same statically bound method can share
4587 // a stub to the interpreter.
4588 __ code()->shared_stub_to_interp_for(_method, call_offset);
4589 } else {
4590 // Emit stubs for static call.
4591 address stub = CompiledDirectCall::emit_to_interp_stub(masm, mark);
4592 __ clear_inst_mark();
4593 if (stub == nullptr) {
4594 ciEnv::current()->record_failure("CodeCache is full");
4595 return;
4596 }
4597 }
4598 }
4599 __ post_call_nop();
4600 %}
4601
4602 enc_class Java_Dynamic_Call(method meth) %{
4603 __ ic_call((address)$meth$$method, resolved_method_index(masm));
4604 __ post_call_nop();
4605 %}
4606
4607 enc_class call_epilog %{
4608 if (VerifyStackAtCalls) {
4609 // Check that stack depth is unchanged: find majik cookie on stack
4610 int framesize = ra_->reg2offset_unchecked(OptoReg::add(ra_->_matcher._old_SP, -3*VMRegImpl::slots_per_word));
4611 Label L;
4612 __ cmpptr(Address(rsp, framesize), (int32_t)0xbadb100d);
4613 __ jccb(Assembler::equal, L);
4614 // Die if stack mismatch
4615 __ int3();
4616 __ bind(L);
4617 }
4618 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
4619 // The last return value is not set by the callee but used to pass the null marker to compiled code.
4620 // Search for the corresponding projection, get the register and emit code that initializes it.
4621 uint con = (tf()->range_cc()->cnt() - 1);
4622 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
4623 ProjNode* proj = fast_out(i)->as_Proj();
4624 if (proj->_con == con) {
4625 // Set null marker if rax is non-null (a non-null value is returned buffered or scalarized)
4626 OptoReg::Name optoReg = ra_->get_reg_first(proj);
4627 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
4628 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
4629 __ testq(rax, rax);
4630 __ setb(Assembler::notZero, toReg);
4631 __ movzbl(toReg, toReg);
4632 if (reg->is_stack()) {
4633 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
4634 __ movq(Address(rsp, st_off), toReg);
4635 }
4636 break;
4637 }
4638 }
4639 if (return_value_is_used()) {
4640 // An inline type is returned as fields in multiple registers.
4641 // Rax either contains an oop if the inline type is buffered or a pointer
4642 // to the corresponding InlineKlass with the lowest bit set to 1. Zero rax
4643 // if the lowest bit is set to allow C2 to use the oop after null checking.
4644 // rax &= (rax & 1) - 1
4645 __ movptr(rscratch1, rax);
4646 __ andptr(rscratch1, 0x1);
4647 __ subptr(rscratch1, 0x1);
4648 __ andptr(rax, rscratch1);
4649 }
4650 }
4651 %}
4652
4653 %}
4654
4655 //----------FRAME--------------------------------------------------------------
4656 // Definition of frame structure and management information.
4657 //
4658 // S T A C K L A Y O U T Allocators stack-slot number
4659 // | (to get allocators register number
4660 // G Owned by | | v add OptoReg::stack0())
4661 // r CALLER | |
4662 // o | +--------+ pad to even-align allocators stack-slot
4663 // w V | pad0 | numbers; owned by CALLER
4664 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
4665 // h ^ | in | 5
4666 // | | args | 4 Holes in incoming args owned by SELF
4667 // | | | | 3
4668 // | | +--------+
4669 // V | | old out| Empty on Intel, window on Sparc
4670 // | old |preserve| Must be even aligned.
4671 // | SP-+--------+----> Matcher::_old_SP, even aligned
4672 // | | in | 3 area for Intel ret address
4673 // Owned by |preserve| Empty on Sparc.
4674 // SELF +--------+
4675 // | | pad2 | 2 pad to align old SP
4676 // | +--------+ 1
4677 // | | locks | 0
4678 // | +--------+----> OptoReg::stack0(), even aligned
4679 // | | pad1 | 11 pad to align new SP
4680 // | +--------+
4681 // | | | 10
4682 // | | spills | 9 spills
4683 // V | | 8 (pad0 slot for callee)
4684 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
4685 // ^ | out | 7
4686 // | | args | 6 Holes in outgoing args owned by CALLEE
4687 // Owned by +--------+
4688 // CALLEE | new out| 6 Empty on Intel, window on Sparc
4689 // | new |preserve| Must be even-aligned.
4690 // | SP-+--------+----> Matcher::_new_SP, even aligned
4691 // | | |
4692 //
4693 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
4694 // known from SELF's arguments and the Java calling convention.
4695 // Region 6-7 is determined per call site.
4696 // Note 2: If the calling convention leaves holes in the incoming argument
4697 // area, those holes are owned by SELF. Holes in the outgoing area
4698 // are owned by the CALLEE. Holes should not be necessary in the
4699 // incoming area, as the Java calling convention is completely under
4700 // the control of the AD file. Doubles can be sorted and packed to
4701 // avoid holes. Holes in the outgoing arguments may be necessary for
4702 // varargs C calling conventions.
4703 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
4704 // even aligned with pad0 as needed.
4705 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
4706 // region 6-11 is even aligned; it may be padded out more so that
4707 // the region from SP to FP meets the minimum stack alignment.
4708 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
4709 // alignment. Region 11, pad1, may be dynamically extended so that
4710 // SP meets the minimum alignment.
4711
4712 frame
4713 %{
4714 // These three registers define part of the calling convention
4715 // between compiled code and the interpreter.
4716 inline_cache_reg(RAX); // Inline Cache Register
4717
4718 // Optional: name the operand used by cisc-spilling to access
4719 // [stack_pointer + offset]
4720 cisc_spilling_operand_name(indOffset32);
4721
4722 // Number of stack slots consumed by locking an object
4723 sync_stack_slots(2);
4724
4725 // Compiled code's Frame Pointer
4726 frame_pointer(RSP);
4727
4728 // Stack alignment requirement
4729 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
4730
4731 // Number of outgoing stack slots killed above the out_preserve_stack_slots
4732 // for calls to C. Supports the var-args backing area for register parms.
4733 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
4734
4735 // The after-PROLOG location of the return address. Location of
4736 // return address specifies a type (REG or STACK) and a number
4737 // representing the register number (i.e. - use a register name) or
4738 // stack slot.
4739 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4740 // Otherwise, it is above the locks and verification slot and alignment word
4741 return_addr(STACK - 2 +
4742 align_up((Compile::current()->in_preserve_stack_slots() +
4743 Compile::current()->fixed_slots()),
4744 stack_alignment_in_slots()));
4745
4746 // Location of compiled Java return values. Same as C for now.
4747 return_value
4748 %{
4749 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4750 "only return normal values");
4751
4752 static const int lo[Op_RegL + 1] = {
4753 0,
4754 0,
4755 RAX_num, // Op_RegN
4756 RAX_num, // Op_RegI
4757 RAX_num, // Op_RegP
4758 XMM0_num, // Op_RegF
4759 XMM0_num, // Op_RegD
4760 RAX_num // Op_RegL
4761 };
4762 static const int hi[Op_RegL + 1] = {
4763 0,
4764 0,
4765 OptoReg::Bad, // Op_RegN
4766 OptoReg::Bad, // Op_RegI
4767 RAX_H_num, // Op_RegP
4768 OptoReg::Bad, // Op_RegF
4769 XMM0b_num, // Op_RegD
4770 RAX_H_num // Op_RegL
4771 };
4772 // Excluded flags and vector registers.
4773 assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
4774 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4775 %}
4776 %}
4777
4778 //----------ATTRIBUTES---------------------------------------------------------
4779 //----------Operand Attributes-------------------------------------------------
4780 op_attrib op_cost(0); // Required cost attribute
4781
4782 //----------Instruction Attributes---------------------------------------------
4783 ins_attrib ins_cost(100); // Required cost attribute
4784 ins_attrib ins_size(8); // Required size attribute (in bits)
4785 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4786 // a non-matching short branch variant
4787 // of some long branch?
4788 ins_attrib ins_alignment(1); // Required alignment attribute (must
4789 // be a power of 2) specifies the
4790 // alignment that some part of the
4791 // instruction (not necessarily the
4792 // start) requires. If > 1, a
4793 // compute_padding() function must be
4794 // provided for the instruction
4795
4796 // Whether this node is expanded during code emission into a sequence of
4797 // instructions and the first instruction can perform an implicit null check.
4798 ins_attrib ins_is_late_expanded_null_check_candidate(false);
4799
4800 //----------OPERANDS-----------------------------------------------------------
4801 // Operand definitions must precede instruction definitions for correct parsing
4802 // in the ADLC because operands constitute user defined types which are used in
4803 // instruction definitions.
4804
4805 //----------Simple Operands----------------------------------------------------
4806 // Immediate Operands
4807 // Integer Immediate
4808 operand immI()
4809 %{
4810 match(ConI);
4811
4812 op_cost(10);
4813 format %{ %}
4814 interface(CONST_INTER);
4815 %}
4816
4817 // Constant for test vs zero
4818 operand immI_0()
4819 %{
4820 predicate(n->get_int() == 0);
4821 match(ConI);
4822
4823 op_cost(0);
4824 format %{ %}
4825 interface(CONST_INTER);
4826 %}
4827
4828 // Constant for increment
4829 operand immI_1()
4830 %{
4831 predicate(n->get_int() == 1);
4832 match(ConI);
4833
4834 op_cost(0);
4835 format %{ %}
4836 interface(CONST_INTER);
4837 %}
4838
4839 // Constant for decrement
4840 operand immI_M1()
4841 %{
4842 predicate(n->get_int() == -1);
4843 match(ConI);
4844
4845 op_cost(0);
4846 format %{ %}
4847 interface(CONST_INTER);
4848 %}
4849
4850 operand immI_2()
4851 %{
4852 predicate(n->get_int() == 2);
4853 match(ConI);
4854
4855 op_cost(0);
4856 format %{ %}
4857 interface(CONST_INTER);
4858 %}
4859
4860 operand immI_4()
4861 %{
4862 predicate(n->get_int() == 4);
4863 match(ConI);
4864
4865 op_cost(0);
4866 format %{ %}
4867 interface(CONST_INTER);
4868 %}
4869
4870 operand immI_8()
4871 %{
4872 predicate(n->get_int() == 8);
4873 match(ConI);
4874
4875 op_cost(0);
4876 format %{ %}
4877 interface(CONST_INTER);
4878 %}
4879
4880 // Valid scale values for addressing modes
4881 operand immI2()
4882 %{
4883 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4884 match(ConI);
4885
4886 format %{ %}
4887 interface(CONST_INTER);
4888 %}
4889
4890 operand immU7()
4891 %{
4892 predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
4893 match(ConI);
4894
4895 op_cost(5);
4896 format %{ %}
4897 interface(CONST_INTER);
4898 %}
4899
4900 operand immI8()
4901 %{
4902 predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
4903 match(ConI);
4904
4905 op_cost(5);
4906 format %{ %}
4907 interface(CONST_INTER);
4908 %}
4909
4910 operand immU8()
4911 %{
4912 predicate((0 <= n->get_int()) && (n->get_int() <= 255));
4913 match(ConI);
4914
4915 op_cost(5);
4916 format %{ %}
4917 interface(CONST_INTER);
4918 %}
4919
4920 operand immI16()
4921 %{
4922 predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
4923 match(ConI);
4924
4925 op_cost(10);
4926 format %{ %}
4927 interface(CONST_INTER);
4928 %}
4929
4930 // Int Immediate non-negative
4931 operand immU31()
4932 %{
4933 predicate(n->get_int() >= 0);
4934 match(ConI);
4935
4936 op_cost(0);
4937 format %{ %}
4938 interface(CONST_INTER);
4939 %}
4940
4941 // Pointer Immediate
4942 operand immP()
4943 %{
4944 match(ConP);
4945
4946 op_cost(10);
4947 format %{ %}
4948 interface(CONST_INTER);
4949 %}
4950
4951 // Null Pointer Immediate
4952 operand immP0()
4953 %{
4954 predicate(n->get_ptr() == 0);
4955 match(ConP);
4956
4957 op_cost(5);
4958 format %{ %}
4959 interface(CONST_INTER);
4960 %}
4961
4962 // Pointer Immediate
4963 operand immN() %{
4964 match(ConN);
4965
4966 op_cost(10);
4967 format %{ %}
4968 interface(CONST_INTER);
4969 %}
4970
4971 operand immNKlass() %{
4972 match(ConNKlass);
4973
4974 op_cost(10);
4975 format %{ %}
4976 interface(CONST_INTER);
4977 %}
4978
4979 // Null Pointer Immediate
4980 operand immN0() %{
4981 predicate(n->get_narrowcon() == 0);
4982 match(ConN);
4983
4984 op_cost(5);
4985 format %{ %}
4986 interface(CONST_INTER);
4987 %}
4988
4989 operand immP31()
4990 %{
4991 predicate(n->as_Type()->type()->is_ptr()->reloc() == relocInfo::none
4992 && (n->get_ptr() >> 31) == 0);
4993 match(ConP);
4994
4995 op_cost(5);
4996 format %{ %}
4997 interface(CONST_INTER);
4998 %}
4999
5000
5001 // Long Immediate
5002 operand immL()
5003 %{
5004 match(ConL);
5005
5006 op_cost(20);
5007 format %{ %}
5008 interface(CONST_INTER);
5009 %}
5010
5011 // Long Immediate 8-bit
5012 operand immL8()
5013 %{
5014 predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
5015 match(ConL);
5016
5017 op_cost(5);
5018 format %{ %}
5019 interface(CONST_INTER);
5020 %}
5021
5022 // Long Immediate 32-bit unsigned
5023 operand immUL32()
5024 %{
5025 predicate(n->get_long() == (unsigned int) (n->get_long()));
5026 match(ConL);
5027
5028 op_cost(10);
5029 format %{ %}
5030 interface(CONST_INTER);
5031 %}
5032
5033 // Long Immediate 32-bit signed
5034 operand immL32()
5035 %{
5036 predicate(n->get_long() == (int) (n->get_long()));
5037 match(ConL);
5038
5039 op_cost(15);
5040 format %{ %}
5041 interface(CONST_INTER);
5042 %}
5043
5044 operand immL_Pow2()
5045 %{
5046 predicate(is_power_of_2((julong)n->get_long()));
5047 match(ConL);
5048
5049 op_cost(15);
5050 format %{ %}
5051 interface(CONST_INTER);
5052 %}
5053
5054 operand immL_NotPow2()
5055 %{
5056 predicate(is_power_of_2((julong)~n->get_long()));
5057 match(ConL);
5058
5059 op_cost(15);
5060 format %{ %}
5061 interface(CONST_INTER);
5062 %}
5063
5064 // Long Immediate zero
5065 operand immL0()
5066 %{
5067 predicate(n->get_long() == 0L);
5068 match(ConL);
5069
5070 op_cost(10);
5071 format %{ %}
5072 interface(CONST_INTER);
5073 %}
5074
5075 // Constant for increment
5076 operand immL1()
5077 %{
5078 predicate(n->get_long() == 1);
5079 match(ConL);
5080
5081 format %{ %}
5082 interface(CONST_INTER);
5083 %}
5084
5085 // Constant for decrement
5086 operand immL_M1()
5087 %{
5088 predicate(n->get_long() == -1);
5089 match(ConL);
5090
5091 format %{ %}
5092 interface(CONST_INTER);
5093 %}
5094
5095 // Long Immediate: low 32-bit mask
5096 operand immL_32bits()
5097 %{
5098 predicate(n->get_long() == 0xFFFFFFFFL);
5099 match(ConL);
5100 op_cost(20);
5101
5102 format %{ %}
5103 interface(CONST_INTER);
5104 %}
5105
5106 // Int Immediate: 2^n-1, positive
5107 operand immI_Pow2M1()
5108 %{
5109 predicate((n->get_int() > 0)
5110 && is_power_of_2((juint)n->get_int() + 1));
5111 match(ConI);
5112
5113 op_cost(20);
5114 format %{ %}
5115 interface(CONST_INTER);
5116 %}
5117
5118 // Float Immediate zero
5119 operand immF0()
5120 %{
5121 predicate(jint_cast(n->getf()) == 0);
5122 match(ConF);
5123
5124 op_cost(5);
5125 format %{ %}
5126 interface(CONST_INTER);
5127 %}
5128
5129 // Float Immediate
5130 operand immF()
5131 %{
5132 match(ConF);
5133
5134 op_cost(15);
5135 format %{ %}
5136 interface(CONST_INTER);
5137 %}
5138
5139 // Half Float Immediate
5140 operand immH()
5141 %{
5142 match(ConH);
5143
5144 op_cost(15);
5145 format %{ %}
5146 interface(CONST_INTER);
5147 %}
5148
5149 // Double Immediate zero
5150 operand immD0()
5151 %{
5152 predicate(jlong_cast(n->getd()) == 0);
5153 match(ConD);
5154
5155 op_cost(5);
5156 format %{ %}
5157 interface(CONST_INTER);
5158 %}
5159
5160 // Double Immediate
5161 operand immD()
5162 %{
5163 match(ConD);
5164
5165 op_cost(15);
5166 format %{ %}
5167 interface(CONST_INTER);
5168 %}
5169
5170 // Immediates for special shifts (sign extend)
5171
5172 // Constants for increment
5173 operand immI_16()
5174 %{
5175 predicate(n->get_int() == 16);
5176 match(ConI);
5177
5178 format %{ %}
5179 interface(CONST_INTER);
5180 %}
5181
5182 operand immI_24()
5183 %{
5184 predicate(n->get_int() == 24);
5185 match(ConI);
5186
5187 format %{ %}
5188 interface(CONST_INTER);
5189 %}
5190
5191 // Constant for byte-wide masking
5192 operand immI_255()
5193 %{
5194 predicate(n->get_int() == 255);
5195 match(ConI);
5196
5197 format %{ %}
5198 interface(CONST_INTER);
5199 %}
5200
5201 // Constant for short-wide masking
5202 operand immI_65535()
5203 %{
5204 predicate(n->get_int() == 65535);
5205 match(ConI);
5206
5207 format %{ %}
5208 interface(CONST_INTER);
5209 %}
5210
5211 // Constant for byte-wide masking
5212 operand immL_255()
5213 %{
5214 predicate(n->get_long() == 255);
5215 match(ConL);
5216
5217 format %{ %}
5218 interface(CONST_INTER);
5219 %}
5220
5221 // Constant for short-wide masking
5222 operand immL_65535()
5223 %{
5224 predicate(n->get_long() == 65535);
5225 match(ConL);
5226
5227 format %{ %}
5228 interface(CONST_INTER);
5229 %}
5230
5231 // AOT Runtime Constants Address
5232 operand immAOTRuntimeConstantsAddress()
5233 %{
5234 // Check if the address is in the range of AOT Runtime Constants
5235 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
5236 match(ConP);
5237
5238 op_cost(0);
5239 format %{ %}
5240 interface(CONST_INTER);
5241 %}
5242
5243 operand kReg()
5244 %{
5245 constraint(ALLOC_IN_RC(vectmask_reg));
5246 match(RegVectMask);
5247 format %{%}
5248 interface(REG_INTER);
5249 %}
5250
5251 // Register Operands
5252 // Integer Register
5253 operand rRegI()
5254 %{
5255 constraint(ALLOC_IN_RC(int_reg));
5256 match(RegI);
5257
5258 match(rax_RegI);
5259 match(rbx_RegI);
5260 match(rcx_RegI);
5261 match(rdx_RegI);
5262 match(rdi_RegI);
5263
5264 format %{ %}
5265 interface(REG_INTER);
5266 %}
5267
5268 // Special Registers
5269 operand rax_RegI()
5270 %{
5271 constraint(ALLOC_IN_RC(int_rax_reg));
5272 match(RegI);
5273 match(rRegI);
5274
5275 format %{ "RAX" %}
5276 interface(REG_INTER);
5277 %}
5278
5279 // Special Registers
5280 operand rbx_RegI()
5281 %{
5282 constraint(ALLOC_IN_RC(int_rbx_reg));
5283 match(RegI);
5284 match(rRegI);
5285
5286 format %{ "RBX" %}
5287 interface(REG_INTER);
5288 %}
5289
5290 operand rcx_RegI()
5291 %{
5292 constraint(ALLOC_IN_RC(int_rcx_reg));
5293 match(RegI);
5294 match(rRegI);
5295
5296 format %{ "RCX" %}
5297 interface(REG_INTER);
5298 %}
5299
5300 operand rdx_RegI()
5301 %{
5302 constraint(ALLOC_IN_RC(int_rdx_reg));
5303 match(RegI);
5304 match(rRegI);
5305
5306 format %{ "RDX" %}
5307 interface(REG_INTER);
5308 %}
5309
5310 operand rdi_RegI()
5311 %{
5312 constraint(ALLOC_IN_RC(int_rdi_reg));
5313 match(RegI);
5314 match(rRegI);
5315
5316 format %{ "RDI" %}
5317 interface(REG_INTER);
5318 %}
5319
5320 operand no_rax_rdx_RegI()
5321 %{
5322 constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
5323 match(RegI);
5324 match(rbx_RegI);
5325 match(rcx_RegI);
5326 match(rdi_RegI);
5327
5328 format %{ %}
5329 interface(REG_INTER);
5330 %}
5331
5332 operand no_rbp_r13_RegI()
5333 %{
5334 constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
5335 match(RegI);
5336 match(rRegI);
5337 match(rax_RegI);
5338 match(rbx_RegI);
5339 match(rcx_RegI);
5340 match(rdx_RegI);
5341 match(rdi_RegI);
5342
5343 format %{ %}
5344 interface(REG_INTER);
5345 %}
5346
5347 // Pointer Register
5348 operand any_RegP()
5349 %{
5350 constraint(ALLOC_IN_RC(any_reg));
5351 match(RegP);
5352 match(rax_RegP);
5353 match(rbx_RegP);
5354 match(rdi_RegP);
5355 match(rsi_RegP);
5356 match(rbp_RegP);
5357 match(r15_RegP);
5358 match(rRegP);
5359
5360 format %{ %}
5361 interface(REG_INTER);
5362 %}
5363
5364 operand rRegP()
5365 %{
5366 constraint(ALLOC_IN_RC(ptr_reg));
5367 match(RegP);
5368 match(rax_RegP);
5369 match(rbx_RegP);
5370 match(rdi_RegP);
5371 match(rsi_RegP);
5372 match(rbp_RegP); // See Q&A below about
5373 match(r15_RegP); // r15_RegP and rbp_RegP.
5374
5375 format %{ %}
5376 interface(REG_INTER);
5377 %}
5378
5379 operand rRegN() %{
5380 constraint(ALLOC_IN_RC(int_reg));
5381 match(RegN);
5382
5383 format %{ %}
5384 interface(REG_INTER);
5385 %}
5386
5387 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
5388 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
5389 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
5390 // The output of an instruction is controlled by the allocator, which respects
5391 // register class masks, not match rules. Unless an instruction mentions
5392 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
5393 // by the allocator as an input.
5394 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
5395 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
5396 // result, RBP is not included in the output of the instruction either.
5397
5398 // This operand is not allowed to use RBP even if
5399 // RBP is not used to hold the frame pointer.
5400 operand no_rbp_RegP()
5401 %{
5402 constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
5403 match(RegP);
5404 match(rbx_RegP);
5405 match(rsi_RegP);
5406 match(rdi_RegP);
5407
5408 format %{ %}
5409 interface(REG_INTER);
5410 %}
5411
5412 // Special Registers
5413 // Return a pointer value
5414 operand rax_RegP()
5415 %{
5416 constraint(ALLOC_IN_RC(ptr_rax_reg));
5417 match(RegP);
5418 match(rRegP);
5419
5420 format %{ %}
5421 interface(REG_INTER);
5422 %}
5423
5424 // Special Registers
5425 // Return a compressed pointer value
5426 operand rax_RegN()
5427 %{
5428 constraint(ALLOC_IN_RC(int_rax_reg));
5429 match(RegN);
5430 match(rRegN);
5431
5432 format %{ %}
5433 interface(REG_INTER);
5434 %}
5435
5436 // Used in AtomicAdd
5437 operand rbx_RegP()
5438 %{
5439 constraint(ALLOC_IN_RC(ptr_rbx_reg));
5440 match(RegP);
5441 match(rRegP);
5442
5443 format %{ %}
5444 interface(REG_INTER);
5445 %}
5446
5447 operand rsi_RegP()
5448 %{
5449 constraint(ALLOC_IN_RC(ptr_rsi_reg));
5450 match(RegP);
5451 match(rRegP);
5452
5453 format %{ %}
5454 interface(REG_INTER);
5455 %}
5456
5457 operand rbp_RegP()
5458 %{
5459 constraint(ALLOC_IN_RC(ptr_rbp_reg));
5460 match(RegP);
5461 match(rRegP);
5462
5463 format %{ %}
5464 interface(REG_INTER);
5465 %}
5466
5467 // Used in rep stosq
5468 operand rdi_RegP()
5469 %{
5470 constraint(ALLOC_IN_RC(ptr_rdi_reg));
5471 match(RegP);
5472 match(rRegP);
5473
5474 format %{ %}
5475 interface(REG_INTER);
5476 %}
5477
5478 operand r15_RegP()
5479 %{
5480 constraint(ALLOC_IN_RC(ptr_r15_reg));
5481 match(RegP);
5482 match(rRegP);
5483
5484 format %{ %}
5485 interface(REG_INTER);
5486 %}
5487
5488 operand rRegL()
5489 %{
5490 constraint(ALLOC_IN_RC(long_reg));
5491 match(RegL);
5492 match(rax_RegL);
5493 match(rdx_RegL);
5494
5495 format %{ %}
5496 interface(REG_INTER);
5497 %}
5498
5499 // Special Registers
5500 operand no_rax_rdx_RegL()
5501 %{
5502 constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
5503 match(RegL);
5504 match(rRegL);
5505
5506 format %{ %}
5507 interface(REG_INTER);
5508 %}
5509
5510 operand rax_RegL()
5511 %{
5512 constraint(ALLOC_IN_RC(long_rax_reg));
5513 match(RegL);
5514 match(rRegL);
5515
5516 format %{ "RAX" %}
5517 interface(REG_INTER);
5518 %}
5519
5520 operand rcx_RegL()
5521 %{
5522 constraint(ALLOC_IN_RC(long_rcx_reg));
5523 match(RegL);
5524 match(rRegL);
5525
5526 format %{ %}
5527 interface(REG_INTER);
5528 %}
5529
5530 operand rdx_RegL()
5531 %{
5532 constraint(ALLOC_IN_RC(long_rdx_reg));
5533 match(RegL);
5534 match(rRegL);
5535
5536 format %{ %}
5537 interface(REG_INTER);
5538 %}
5539
5540 operand r11_RegL()
5541 %{
5542 constraint(ALLOC_IN_RC(long_r11_reg));
5543 match(RegL);
5544 match(rRegL);
5545
5546 format %{ %}
5547 interface(REG_INTER);
5548 %}
5549
5550 operand no_rbp_r13_RegL()
5551 %{
5552 constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
5553 match(RegL);
5554 match(rRegL);
5555 match(rax_RegL);
5556 match(rcx_RegL);
5557 match(rdx_RegL);
5558
5559 format %{ %}
5560 interface(REG_INTER);
5561 %}
5562
5563 // Flags register, used as output of compare instructions
5564 operand rFlagsReg()
5565 %{
5566 constraint(ALLOC_IN_RC(int_flags));
5567 match(RegFlags);
5568
5569 format %{ "RFLAGS" %}
5570 interface(REG_INTER);
5571 %}
5572
5573 // Flags register, used as output of FLOATING POINT compare instructions
5574 operand rFlagsRegU()
5575 %{
5576 constraint(ALLOC_IN_RC(int_flags));
5577 match(RegFlags);
5578
5579 format %{ "RFLAGS_U" %}
5580 interface(REG_INTER);
5581 %}
5582
5583 operand rFlagsRegUCF() %{
5584 constraint(ALLOC_IN_RC(int_flags));
5585 match(RegFlags);
5586 predicate(!UseAPX || !VM_Version::supports_avx10_2());
5587
5588 format %{ "RFLAGS_U_CF" %}
5589 interface(REG_INTER);
5590 %}
5591
5592 operand rFlagsRegUCFE() %{
5593 constraint(ALLOC_IN_RC(int_flags));
5594 match(RegFlags);
5595 predicate(UseAPX && VM_Version::supports_avx10_2());
5596
5597 format %{ "RFLAGS_U_CFE" %}
5598 interface(REG_INTER);
5599 %}
5600
5601 // Float register operands
5602 operand regF() %{
5603 constraint(ALLOC_IN_RC(float_reg));
5604 match(RegF);
5605
5606 format %{ %}
5607 interface(REG_INTER);
5608 %}
5609
5610 // Float register operands
5611 operand legRegF() %{
5612 constraint(ALLOC_IN_RC(float_reg_legacy));
5613 match(RegF);
5614
5615 format %{ %}
5616 interface(REG_INTER);
5617 %}
5618
5619 // Float register operands
5620 operand vlRegF() %{
5621 constraint(ALLOC_IN_RC(float_reg_vl));
5622 match(RegF);
5623
5624 format %{ %}
5625 interface(REG_INTER);
5626 %}
5627
5628 // Double register operands
5629 operand regD() %{
5630 constraint(ALLOC_IN_RC(double_reg));
5631 match(RegD);
5632
5633 format %{ %}
5634 interface(REG_INTER);
5635 %}
5636
5637 // Double register operands
5638 operand legRegD() %{
5639 constraint(ALLOC_IN_RC(double_reg_legacy));
5640 match(RegD);
5641
5642 format %{ %}
5643 interface(REG_INTER);
5644 %}
5645
5646 // Double register operands
5647 operand vlRegD() %{
5648 constraint(ALLOC_IN_RC(double_reg_vl));
5649 match(RegD);
5650
5651 format %{ %}
5652 interface(REG_INTER);
5653 %}
5654
5655 //----------Memory Operands----------------------------------------------------
5656 // Direct Memory Operand
5657 // operand direct(immP addr)
5658 // %{
5659 // match(addr);
5660
5661 // format %{ "[$addr]" %}
5662 // interface(MEMORY_INTER) %{
5663 // base(0xFFFFFFFF);
5664 // index(0x4);
5665 // scale(0x0);
5666 // disp($addr);
5667 // %}
5668 // %}
5669
5670 // Indirect Memory Operand
5671 operand indirect(any_RegP reg)
5672 %{
5673 constraint(ALLOC_IN_RC(ptr_reg));
5674 match(reg);
5675
5676 format %{ "[$reg]" %}
5677 interface(MEMORY_INTER) %{
5678 base($reg);
5679 index(0x4);
5680 scale(0x0);
5681 disp(0x0);
5682 %}
5683 %}
5684
5685 // Indirect Memory Plus Short Offset Operand
5686 operand indOffset8(any_RegP reg, immL8 off)
5687 %{
5688 constraint(ALLOC_IN_RC(ptr_reg));
5689 match(AddP reg off);
5690
5691 format %{ "[$reg + $off (8-bit)]" %}
5692 interface(MEMORY_INTER) %{
5693 base($reg);
5694 index(0x4);
5695 scale(0x0);
5696 disp($off);
5697 %}
5698 %}
5699
5700 // Indirect Memory Plus Long Offset Operand
5701 operand indOffset32(any_RegP reg, immL32 off)
5702 %{
5703 constraint(ALLOC_IN_RC(ptr_reg));
5704 match(AddP reg off);
5705
5706 format %{ "[$reg + $off (32-bit)]" %}
5707 interface(MEMORY_INTER) %{
5708 base($reg);
5709 index(0x4);
5710 scale(0x0);
5711 disp($off);
5712 %}
5713 %}
5714
5715 // Indirect Memory Plus Index Register Plus Offset Operand
5716 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
5717 %{
5718 constraint(ALLOC_IN_RC(ptr_reg));
5719 match(AddP (AddP reg lreg) off);
5720
5721 op_cost(10);
5722 format %{"[$reg + $off + $lreg]" %}
5723 interface(MEMORY_INTER) %{
5724 base($reg);
5725 index($lreg);
5726 scale(0x0);
5727 disp($off);
5728 %}
5729 %}
5730
5731 // Indirect Memory Plus Index Register Plus Offset Operand
5732 operand indIndex(any_RegP reg, rRegL lreg)
5733 %{
5734 constraint(ALLOC_IN_RC(ptr_reg));
5735 match(AddP reg lreg);
5736
5737 op_cost(10);
5738 format %{"[$reg + $lreg]" %}
5739 interface(MEMORY_INTER) %{
5740 base($reg);
5741 index($lreg);
5742 scale(0x0);
5743 disp(0x0);
5744 %}
5745 %}
5746
5747 // Indirect Memory Times Scale Plus Index Register
5748 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
5749 %{
5750 constraint(ALLOC_IN_RC(ptr_reg));
5751 match(AddP reg (LShiftL lreg scale));
5752
5753 op_cost(10);
5754 format %{"[$reg + $lreg << $scale]" %}
5755 interface(MEMORY_INTER) %{
5756 base($reg);
5757 index($lreg);
5758 scale($scale);
5759 disp(0x0);
5760 %}
5761 %}
5762
5763 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
5764 %{
5765 constraint(ALLOC_IN_RC(ptr_reg));
5766 predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
5767 match(AddP reg (LShiftL (ConvI2L idx) scale));
5768
5769 op_cost(10);
5770 format %{"[$reg + pos $idx << $scale]" %}
5771 interface(MEMORY_INTER) %{
5772 base($reg);
5773 index($idx);
5774 scale($scale);
5775 disp(0x0);
5776 %}
5777 %}
5778
5779 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
5780 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
5781 %{
5782 constraint(ALLOC_IN_RC(ptr_reg));
5783 match(AddP (AddP reg (LShiftL lreg scale)) off);
5784
5785 op_cost(10);
5786 format %{"[$reg + $off + $lreg << $scale]" %}
5787 interface(MEMORY_INTER) %{
5788 base($reg);
5789 index($lreg);
5790 scale($scale);
5791 disp($off);
5792 %}
5793 %}
5794
5795 // Indirect Memory Plus Positive Index Register Plus Offset Operand
5796 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
5797 %{
5798 constraint(ALLOC_IN_RC(ptr_reg));
5799 predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
5800 match(AddP (AddP reg (ConvI2L idx)) off);
5801
5802 op_cost(10);
5803 format %{"[$reg + $off + $idx]" %}
5804 interface(MEMORY_INTER) %{
5805 base($reg);
5806 index($idx);
5807 scale(0x0);
5808 disp($off);
5809 %}
5810 %}
5811
5812 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
5813 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
5814 %{
5815 constraint(ALLOC_IN_RC(ptr_reg));
5816 predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
5817 match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
5818
5819 op_cost(10);
5820 format %{"[$reg + $off + $idx << $scale]" %}
5821 interface(MEMORY_INTER) %{
5822 base($reg);
5823 index($idx);
5824 scale($scale);
5825 disp($off);
5826 %}
5827 %}
5828
5829 // Indirect Narrow Oop Operand
5830 operand indCompressedOop(rRegN reg) %{
5831 predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
5832 constraint(ALLOC_IN_RC(ptr_reg));
5833 match(DecodeN reg);
5834
5835 op_cost(10);
5836 format %{"[R12 + $reg << 3] (compressed oop addressing)" %}
5837 interface(MEMORY_INTER) %{
5838 base(0xc); // R12
5839 index($reg);
5840 scale(0x3);
5841 disp(0x0);
5842 %}
5843 %}
5844
5845 // Indirect Narrow Oop Plus Offset Operand
5846 // Note: x86 architecture doesn't support "scale * index + offset" without a base
5847 // we can't free r12 even with CompressedOops::base() == nullptr.
5848 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
5849 predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
5850 constraint(ALLOC_IN_RC(ptr_reg));
5851 match(AddP (DecodeN reg) off);
5852
5853 op_cost(10);
5854 format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
5855 interface(MEMORY_INTER) %{
5856 base(0xc); // R12
5857 index($reg);
5858 scale(0x3);
5859 disp($off);
5860 %}
5861 %}
5862
5863 // Indirect Memory Operand
5864 operand indirectNarrow(rRegN reg)
5865 %{
5866 predicate(CompressedOops::shift() == 0);
5867 constraint(ALLOC_IN_RC(ptr_reg));
5868 match(DecodeN reg);
5869
5870 format %{ "[$reg]" %}
5871 interface(MEMORY_INTER) %{
5872 base($reg);
5873 index(0x4);
5874 scale(0x0);
5875 disp(0x0);
5876 %}
5877 %}
5878
5879 // Indirect Memory Plus Short Offset Operand
5880 operand indOffset8Narrow(rRegN reg, immL8 off)
5881 %{
5882 predicate(CompressedOops::shift() == 0);
5883 constraint(ALLOC_IN_RC(ptr_reg));
5884 match(AddP (DecodeN reg) off);
5885
5886 format %{ "[$reg + $off (8-bit)]" %}
5887 interface(MEMORY_INTER) %{
5888 base($reg);
5889 index(0x4);
5890 scale(0x0);
5891 disp($off);
5892 %}
5893 %}
5894
5895 // Indirect Memory Plus Long Offset Operand
5896 operand indOffset32Narrow(rRegN reg, immL32 off)
5897 %{
5898 predicate(CompressedOops::shift() == 0);
5899 constraint(ALLOC_IN_RC(ptr_reg));
5900 match(AddP (DecodeN reg) off);
5901
5902 format %{ "[$reg + $off (32-bit)]" %}
5903 interface(MEMORY_INTER) %{
5904 base($reg);
5905 index(0x4);
5906 scale(0x0);
5907 disp($off);
5908 %}
5909 %}
5910
5911 // Indirect Memory Plus Index Register Plus Offset Operand
5912 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
5913 %{
5914 predicate(CompressedOops::shift() == 0);
5915 constraint(ALLOC_IN_RC(ptr_reg));
5916 match(AddP (AddP (DecodeN reg) lreg) off);
5917
5918 op_cost(10);
5919 format %{"[$reg + $off + $lreg]" %}
5920 interface(MEMORY_INTER) %{
5921 base($reg);
5922 index($lreg);
5923 scale(0x0);
5924 disp($off);
5925 %}
5926 %}
5927
5928 // Indirect Memory Plus Index Register Plus Offset Operand
5929 operand indIndexNarrow(rRegN reg, rRegL lreg)
5930 %{
5931 predicate(CompressedOops::shift() == 0);
5932 constraint(ALLOC_IN_RC(ptr_reg));
5933 match(AddP (DecodeN reg) lreg);
5934
5935 op_cost(10);
5936 format %{"[$reg + $lreg]" %}
5937 interface(MEMORY_INTER) %{
5938 base($reg);
5939 index($lreg);
5940 scale(0x0);
5941 disp(0x0);
5942 %}
5943 %}
5944
5945 // Indirect Memory Times Scale Plus Index Register
5946 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
5947 %{
5948 predicate(CompressedOops::shift() == 0);
5949 constraint(ALLOC_IN_RC(ptr_reg));
5950 match(AddP (DecodeN reg) (LShiftL lreg scale));
5951
5952 op_cost(10);
5953 format %{"[$reg + $lreg << $scale]" %}
5954 interface(MEMORY_INTER) %{
5955 base($reg);
5956 index($lreg);
5957 scale($scale);
5958 disp(0x0);
5959 %}
5960 %}
5961
5962 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
5963 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
5964 %{
5965 predicate(CompressedOops::shift() == 0);
5966 constraint(ALLOC_IN_RC(ptr_reg));
5967 match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
5968
5969 op_cost(10);
5970 format %{"[$reg + $off + $lreg << $scale]" %}
5971 interface(MEMORY_INTER) %{
5972 base($reg);
5973 index($lreg);
5974 scale($scale);
5975 disp($off);
5976 %}
5977 %}
5978
5979 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
5980 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
5981 %{
5982 constraint(ALLOC_IN_RC(ptr_reg));
5983 predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
5984 match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
5985
5986 op_cost(10);
5987 format %{"[$reg + $off + $idx]" %}
5988 interface(MEMORY_INTER) %{
5989 base($reg);
5990 index($idx);
5991 scale(0x0);
5992 disp($off);
5993 %}
5994 %}
5995
5996 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
5997 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
5998 %{
5999 constraint(ALLOC_IN_RC(ptr_reg));
6000 predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
6001 match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
6002
6003 op_cost(10);
6004 format %{"[$reg + $off + $idx << $scale]" %}
6005 interface(MEMORY_INTER) %{
6006 base($reg);
6007 index($idx);
6008 scale($scale);
6009 disp($off);
6010 %}
6011 %}
6012
6013 //----------Special Memory Operands--------------------------------------------
6014 // Stack Slot Operand - This operand is used for loading and storing temporary
6015 // values on the stack where a match requires a value to
6016 // flow through memory.
6017 operand stackSlotP(sRegP reg)
6018 %{
6019 constraint(ALLOC_IN_RC(stack_slots));
6020 // No match rule because this operand is only generated in matching
6021
6022 format %{ "[$reg]" %}
6023 interface(MEMORY_INTER) %{
6024 base(0x4); // RSP
6025 index(0x4); // No Index
6026 scale(0x0); // No Scale
6027 disp($reg); // Stack Offset
6028 %}
6029 %}
6030
6031 operand stackSlotI(sRegI reg)
6032 %{
6033 constraint(ALLOC_IN_RC(stack_slots));
6034 // No match rule because this operand is only generated in matching
6035
6036 format %{ "[$reg]" %}
6037 interface(MEMORY_INTER) %{
6038 base(0x4); // RSP
6039 index(0x4); // No Index
6040 scale(0x0); // No Scale
6041 disp($reg); // Stack Offset
6042 %}
6043 %}
6044
6045 operand stackSlotF(sRegF reg)
6046 %{
6047 constraint(ALLOC_IN_RC(stack_slots));
6048 // No match rule because this operand is only generated in matching
6049
6050 format %{ "[$reg]" %}
6051 interface(MEMORY_INTER) %{
6052 base(0x4); // RSP
6053 index(0x4); // No Index
6054 scale(0x0); // No Scale
6055 disp($reg); // Stack Offset
6056 %}
6057 %}
6058
6059 operand stackSlotD(sRegD reg)
6060 %{
6061 constraint(ALLOC_IN_RC(stack_slots));
6062 // No match rule because this operand is only generated in matching
6063
6064 format %{ "[$reg]" %}
6065 interface(MEMORY_INTER) %{
6066 base(0x4); // RSP
6067 index(0x4); // No Index
6068 scale(0x0); // No Scale
6069 disp($reg); // Stack Offset
6070 %}
6071 %}
6072 operand stackSlotL(sRegL reg)
6073 %{
6074 constraint(ALLOC_IN_RC(stack_slots));
6075 // No match rule because this operand is only generated in matching
6076
6077 format %{ "[$reg]" %}
6078 interface(MEMORY_INTER) %{
6079 base(0x4); // RSP
6080 index(0x4); // No Index
6081 scale(0x0); // No Scale
6082 disp($reg); // Stack Offset
6083 %}
6084 %}
6085
6086 //----------Conditional Branch Operands----------------------------------------
6087 // Comparison Op - This is the operation of the comparison, and is limited to
6088 // the following set of codes:
6089 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6090 //
6091 // Other attributes of the comparison, such as unsignedness, are specified
6092 // by the comparison instruction that sets a condition code flags register.
6093 // That result is represented by a flags operand whose subtype is appropriate
6094 // to the unsignedness (etc.) of the comparison.
6095 //
6096 // Later, the instruction which matches both the Comparison Op (a Bool) and
6097 // the flags (produced by the Cmp) specifies the coding of the comparison op
6098 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6099
6100 // Comparison Code
6101 operand cmpOp()
6102 %{
6103 match(Bool);
6104
6105 format %{ "" %}
6106 interface(COND_INTER) %{
6107 equal(0x4, "e");
6108 not_equal(0x5, "ne");
6109 less(0xc, "l");
6110 greater_equal(0xd, "ge");
6111 less_equal(0xe, "le");
6112 greater(0xf, "g");
6113 overflow(0x0, "o");
6114 no_overflow(0x1, "no");
6115 %}
6116 %}
6117
6118 // Comparison Code, unsigned compare. Used by FP also, with
6119 // C2 (unordered) turned into GT or LT already. The other bits
6120 // C0 and C3 are turned into Carry & Zero flags.
6121 operand cmpOpU()
6122 %{
6123 match(Bool);
6124
6125 format %{ "" %}
6126 interface(COND_INTER) %{
6127 equal(0x4, "e");
6128 not_equal(0x5, "ne");
6129 less(0x2, "b");
6130 greater_equal(0x3, "ae");
6131 less_equal(0x6, "be");
6132 greater(0x7, "a");
6133 overflow(0x0, "o");
6134 no_overflow(0x1, "no");
6135 %}
6136 %}
6137
6138
6139 // Floating comparisons that don't require any fixup for the unordered case,
6140 // If both inputs of the comparison are the same, ZF is always set so we
6141 // don't need to use cmpOpUCF2 for eq/ne
6142 operand cmpOpUCF() %{
6143 match(Bool);
6144 predicate((!UseAPX || !VM_Version::supports_avx10_2()) &&
6145 (n->as_Bool()->_test._test == BoolTest::lt ||
6146 n->as_Bool()->_test._test == BoolTest::ge ||
6147 n->as_Bool()->_test._test == BoolTest::le ||
6148 n->as_Bool()->_test._test == BoolTest::gt ||
6149 n->in(1)->in(1) == n->in(1)->in(2)));
6150 format %{ "" %}
6151 interface(COND_INTER) %{
6152 equal(0xb, "np");
6153 not_equal(0xa, "p");
6154 less(0x2, "b");
6155 greater_equal(0x3, "ae");
6156 less_equal(0x6, "be");
6157 greater(0x7, "a");
6158 overflow(0x0, "o");
6159 no_overflow(0x1, "no");
6160 %}
6161 %}
6162
6163
6164 // Floating comparisons that can be fixed up with extra conditional jumps
6165 operand cmpOpUCF2() %{
6166 match(Bool);
6167 predicate((!UseAPX || !VM_Version::supports_avx10_2()) &&
6168 (n->as_Bool()->_test._test == BoolTest::ne ||
6169 n->as_Bool()->_test._test == BoolTest::eq) &&
6170 n->in(1)->in(1) != n->in(1)->in(2));
6171 format %{ "" %}
6172 interface(COND_INTER) %{
6173 equal(0x4, "e");
6174 not_equal(0x5, "ne");
6175 less(0x2, "b");
6176 greater_equal(0x3, "ae");
6177 less_equal(0x6, "be");
6178 greater(0x7, "a");
6179 overflow(0x0, "o");
6180 no_overflow(0x1, "no");
6181 %}
6182 %}
6183
6184
6185 // Floating point comparisons that set condition flags to test more directly,
6186 // Unsigned tests are used for G (>) and GE (>=) conditions while signed tests
6187 // are used for L (<) and LE (<=) conditions. It's important to convert these
6188 // latter conditions to ones that use unsigned tests before passing into an
6189 // instruction because the preceding comparison might be based on a three way
6190 // comparison (CmpF3 or CmpD3) that also assigns unordered outcomes to -1.
6191 operand cmpOpUCFE()
6192 %{
6193 match(Bool);
6194 predicate((UseAPX && VM_Version::supports_avx10_2()) &&
6195 (n->as_Bool()->_test._test == BoolTest::ne ||
6196 n->as_Bool()->_test._test == BoolTest::eq ||
6197 n->as_Bool()->_test._test == BoolTest::lt ||
6198 n->as_Bool()->_test._test == BoolTest::ge ||
6199 n->as_Bool()->_test._test == BoolTest::le ||
6200 n->as_Bool()->_test._test == BoolTest::gt));
6201
6202 format %{ "" %}
6203 interface(COND_INTER) %{
6204 equal(0x4, "e");
6205 not_equal(0x5, "ne");
6206 less(0x2, "b");
6207 greater_equal(0x3, "ae");
6208 less_equal(0x6, "be");
6209 greater(0x7, "a");
6210 overflow(0x0, "o");
6211 no_overflow(0x1, "no");
6212 %}
6213 %}
6214
6215 // Operands for bound floating pointer register arguments
6216 operand rxmm0() %{
6217 constraint(ALLOC_IN_RC(xmm0_reg));
6218 match(VecX);
6219 format%{%}
6220 interface(REG_INTER);
6221 %}
6222
6223 // Vectors
6224
6225 // Dummy generic vector class. Should be used for all vector operands.
6226 // Replaced with vec[SDXYZ] during post-selection pass.
6227 operand vec() %{
6228 constraint(ALLOC_IN_RC(dynamic));
6229 match(VecX);
6230 match(VecY);
6231 match(VecZ);
6232 match(VecS);
6233 match(VecD);
6234
6235 format %{ %}
6236 interface(REG_INTER);
6237 %}
6238
6239 // Dummy generic legacy vector class. Should be used for all legacy vector operands.
6240 // Replaced with legVec[SDXYZ] during post-selection cleanup.
6241 // Note: legacy register class is used to avoid extra (unneeded in 32-bit VM)
6242 // runtime code generation via reg_class_dynamic.
6243 operand legVec() %{
6244 constraint(ALLOC_IN_RC(dynamic));
6245 match(VecX);
6246 match(VecY);
6247 match(VecZ);
6248 match(VecS);
6249 match(VecD);
6250
6251 format %{ %}
6252 interface(REG_INTER);
6253 %}
6254
6255 // Replaces vec during post-selection cleanup. See above.
6256 operand vecS() %{
6257 constraint(ALLOC_IN_RC(vectors_reg_vlbwdq));
6258 match(VecS);
6259
6260 format %{ %}
6261 interface(REG_INTER);
6262 %}
6263
6264 // Replaces legVec during post-selection cleanup. See above.
6265 operand legVecS() %{
6266 constraint(ALLOC_IN_RC(vectors_reg_legacy));
6267 match(VecS);
6268
6269 format %{ %}
6270 interface(REG_INTER);
6271 %}
6272
6273 // Replaces vec during post-selection cleanup. See above.
6274 operand vecD() %{
6275 constraint(ALLOC_IN_RC(vectord_reg_vlbwdq));
6276 match(VecD);
6277
6278 format %{ %}
6279 interface(REG_INTER);
6280 %}
6281
6282 // Replaces legVec during post-selection cleanup. See above.
6283 operand legVecD() %{
6284 constraint(ALLOC_IN_RC(vectord_reg_legacy));
6285 match(VecD);
6286
6287 format %{ %}
6288 interface(REG_INTER);
6289 %}
6290
6291 // Replaces vec during post-selection cleanup. See above.
6292 operand vecX() %{
6293 constraint(ALLOC_IN_RC(vectorx_reg_vlbwdq));
6294 match(VecX);
6295
6296 format %{ %}
6297 interface(REG_INTER);
6298 %}
6299
6300 // Replaces legVec during post-selection cleanup. See above.
6301 operand legVecX() %{
6302 constraint(ALLOC_IN_RC(vectorx_reg_legacy));
6303 match(VecX);
6304
6305 format %{ %}
6306 interface(REG_INTER);
6307 %}
6308
6309 // Replaces vec during post-selection cleanup. See above.
6310 operand vecY() %{
6311 constraint(ALLOC_IN_RC(vectory_reg_vlbwdq));
6312 match(VecY);
6313
6314 format %{ %}
6315 interface(REG_INTER);
6316 %}
6317
6318 // Replaces legVec during post-selection cleanup. See above.
6319 operand legVecY() %{
6320 constraint(ALLOC_IN_RC(vectory_reg_legacy));
6321 match(VecY);
6322
6323 format %{ %}
6324 interface(REG_INTER);
6325 %}
6326
6327 // Replaces vec during post-selection cleanup. See above.
6328 operand vecZ() %{
6329 constraint(ALLOC_IN_RC(vectorz_reg));
6330 match(VecZ);
6331
6332 format %{ %}
6333 interface(REG_INTER);
6334 %}
6335
6336 // Replaces legVec during post-selection cleanup. See above.
6337 operand legVecZ() %{
6338 constraint(ALLOC_IN_RC(vectorz_reg_legacy));
6339 match(VecZ);
6340
6341 format %{ %}
6342 interface(REG_INTER);
6343 %}
6344
6345 //----------OPERAND CLASSES----------------------------------------------------
6346 // Operand Classes are groups of operands that are used as to simplify
6347 // instruction definitions by not requiring the AD writer to specify separate
6348 // instructions for every form of operand when the instruction accepts
6349 // multiple operand types with the same basic encoding and format. The classic
6350 // case of this is memory operands.
6351
6352 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
6353 indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
6354 indCompressedOop, indCompressedOopOffset,
6355 indirectNarrow, indOffset8Narrow, indOffset32Narrow,
6356 indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
6357 indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
6358
6359 //----------PIPELINE-----------------------------------------------------------
6360 // Rules which define the behavior of the target architectures pipeline.
6361 pipeline %{
6362
6363 //----------ATTRIBUTES---------------------------------------------------------
6364 attributes %{
6365 variable_size_instructions; // Fixed size instructions
6366 max_instructions_per_bundle = 3; // Up to 3 instructions per bundle
6367 instruction_unit_size = 1; // An instruction is 1 bytes long
6368 instruction_fetch_unit_size = 16; // The processor fetches one line
6369 instruction_fetch_units = 1; // of 16 bytes
6370 %}
6371
6372 //----------RESOURCES----------------------------------------------------------
6373 // Resources are the functional units available to the machine
6374
6375 // Generic P2/P3 pipeline
6376 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
6377 // 3 instructions decoded per cycle.
6378 // 2 load/store ops per cycle, 1 branch, 1 FPU,
6379 // 3 ALU op, only ALU0 handles mul instructions.
6380 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
6381 MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
6382 BR, FPU,
6383 ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
6384
6385 //----------PIPELINE DESCRIPTION-----------------------------------------------
6386 // Pipeline Description specifies the stages in the machine's pipeline
6387
6388 // Generic P2/P3 pipeline
6389 pipe_desc(S0, S1, S2, S3, S4, S5);
6390
6391 //----------PIPELINE CLASSES---------------------------------------------------
6392 // Pipeline Classes describe the stages in which input and output are
6393 // referenced by the hardware pipeline.
6394
6395 // Naming convention: ialu or fpu
6396 // Then: _reg
6397 // Then: _reg if there is a 2nd register
6398 // Then: _long if it's a pair of instructions implementing a long
6399 // Then: _fat if it requires the big decoder
6400 // Or: _mem if it requires the big decoder and a memory unit.
6401
6402 // Integer ALU reg operation
6403 pipe_class ialu_reg(rRegI dst)
6404 %{
6405 single_instruction;
6406 dst : S4(write);
6407 dst : S3(read);
6408 DECODE : S0; // any decoder
6409 ALU : S3; // any alu
6410 %}
6411
6412 // Long ALU reg operation
6413 pipe_class ialu_reg_long(rRegL dst)
6414 %{
6415 instruction_count(2);
6416 dst : S4(write);
6417 dst : S3(read);
6418 DECODE : S0(2); // any 2 decoders
6419 ALU : S3(2); // both alus
6420 %}
6421
6422 // Integer ALU reg operation using big decoder
6423 pipe_class ialu_reg_fat(rRegI dst)
6424 %{
6425 single_instruction;
6426 dst : S4(write);
6427 dst : S3(read);
6428 D0 : S0; // big decoder only
6429 ALU : S3; // any alu
6430 %}
6431
6432 // Integer ALU reg-reg operation
6433 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
6434 %{
6435 single_instruction;
6436 dst : S4(write);
6437 src : S3(read);
6438 DECODE : S0; // any decoder
6439 ALU : S3; // any alu
6440 %}
6441
6442 // Integer ALU reg-reg operation
6443 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
6444 %{
6445 single_instruction;
6446 dst : S4(write);
6447 src : S3(read);
6448 D0 : S0; // big decoder only
6449 ALU : S3; // any alu
6450 %}
6451
6452 // Integer ALU reg-mem operation
6453 pipe_class ialu_reg_mem(rRegI dst, memory mem)
6454 %{
6455 single_instruction;
6456 dst : S5(write);
6457 mem : S3(read);
6458 D0 : S0; // big decoder only
6459 ALU : S4; // any alu
6460 MEM : S3; // any mem
6461 %}
6462
6463 // Integer mem operation (prefetch)
6464 pipe_class ialu_mem(memory mem)
6465 %{
6466 single_instruction;
6467 mem : S3(read);
6468 D0 : S0; // big decoder only
6469 MEM : S3; // any mem
6470 %}
6471
6472 // Integer Store to Memory
6473 pipe_class ialu_mem_reg(memory mem, rRegI src)
6474 %{
6475 single_instruction;
6476 mem : S3(read);
6477 src : S5(read);
6478 D0 : S0; // big decoder only
6479 ALU : S4; // any alu
6480 MEM : S3;
6481 %}
6482
6483 // // Long Store to Memory
6484 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
6485 // %{
6486 // instruction_count(2);
6487 // mem : S3(read);
6488 // src : S5(read);
6489 // D0 : S0(2); // big decoder only; twice
6490 // ALU : S4(2); // any 2 alus
6491 // MEM : S3(2); // Both mems
6492 // %}
6493
6494 // Integer Store to Memory
6495 pipe_class ialu_mem_imm(memory mem)
6496 %{
6497 single_instruction;
6498 mem : S3(read);
6499 D0 : S0; // big decoder only
6500 ALU : S4; // any alu
6501 MEM : S3;
6502 %}
6503
6504 // Integer ALU0 reg-reg operation
6505 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
6506 %{
6507 single_instruction;
6508 dst : S4(write);
6509 src : S3(read);
6510 D0 : S0; // Big decoder only
6511 ALU0 : S3; // only alu0
6512 %}
6513
6514 // Integer ALU0 reg-mem operation
6515 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
6516 %{
6517 single_instruction;
6518 dst : S5(write);
6519 mem : S3(read);
6520 D0 : S0; // big decoder only
6521 ALU0 : S4; // ALU0 only
6522 MEM : S3; // any mem
6523 %}
6524
6525 // Integer ALU reg-reg operation
6526 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
6527 %{
6528 single_instruction;
6529 cr : S4(write);
6530 src1 : S3(read);
6531 src2 : S3(read);
6532 DECODE : S0; // any decoder
6533 ALU : S3; // any alu
6534 %}
6535
6536 // Integer ALU reg-imm operation
6537 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
6538 %{
6539 single_instruction;
6540 cr : S4(write);
6541 src1 : S3(read);
6542 DECODE : S0; // any decoder
6543 ALU : S3; // any alu
6544 %}
6545
6546 // Integer ALU reg-mem operation
6547 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
6548 %{
6549 single_instruction;
6550 cr : S4(write);
6551 src1 : S3(read);
6552 src2 : S3(read);
6553 D0 : S0; // big decoder only
6554 ALU : S4; // any alu
6555 MEM : S3;
6556 %}
6557
6558 // Conditional move reg-reg
6559 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
6560 %{
6561 instruction_count(4);
6562 y : S4(read);
6563 q : S3(read);
6564 p : S3(read);
6565 DECODE : S0(4); // any decoder
6566 %}
6567
6568 // Conditional move reg-reg
6569 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
6570 %{
6571 single_instruction;
6572 dst : S4(write);
6573 src : S3(read);
6574 cr : S3(read);
6575 DECODE : S0; // any decoder
6576 %}
6577
6578 // Conditional move reg-mem
6579 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
6580 %{
6581 single_instruction;
6582 dst : S4(write);
6583 src : S3(read);
6584 cr : S3(read);
6585 DECODE : S0; // any decoder
6586 MEM : S3;
6587 %}
6588
6589 // Conditional move reg-reg long
6590 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
6591 %{
6592 single_instruction;
6593 dst : S4(write);
6594 src : S3(read);
6595 cr : S3(read);
6596 DECODE : S0(2); // any 2 decoders
6597 %}
6598
6599 // Float reg-reg operation
6600 pipe_class fpu_reg(regD dst)
6601 %{
6602 instruction_count(2);
6603 dst : S3(read);
6604 DECODE : S0(2); // any 2 decoders
6605 FPU : S3;
6606 %}
6607
6608 // Float reg-reg operation
6609 pipe_class fpu_reg_reg(regD dst, regD src)
6610 %{
6611 instruction_count(2);
6612 dst : S4(write);
6613 src : S3(read);
6614 DECODE : S0(2); // any 2 decoders
6615 FPU : S3;
6616 %}
6617
6618 // Float reg-reg operation
6619 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
6620 %{
6621 instruction_count(3);
6622 dst : S4(write);
6623 src1 : S3(read);
6624 src2 : S3(read);
6625 DECODE : S0(3); // any 3 decoders
6626 FPU : S3(2);
6627 %}
6628
6629 // Float reg-reg operation
6630 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
6631 %{
6632 instruction_count(4);
6633 dst : S4(write);
6634 src1 : S3(read);
6635 src2 : S3(read);
6636 src3 : S3(read);
6637 DECODE : S0(4); // any 3 decoders
6638 FPU : S3(2);
6639 %}
6640
6641 // Float reg-reg operation
6642 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
6643 %{
6644 instruction_count(4);
6645 dst : S4(write);
6646 src1 : S3(read);
6647 src2 : S3(read);
6648 src3 : S3(read);
6649 DECODE : S1(3); // any 3 decoders
6650 D0 : S0; // Big decoder only
6651 FPU : S3(2);
6652 MEM : S3;
6653 %}
6654
6655 // Float reg-mem operation
6656 pipe_class fpu_reg_mem(regD dst, memory mem)
6657 %{
6658 instruction_count(2);
6659 dst : S5(write);
6660 mem : S3(read);
6661 D0 : S0; // big decoder only
6662 DECODE : S1; // any decoder for FPU POP
6663 FPU : S4;
6664 MEM : S3; // any mem
6665 %}
6666
6667 // Float reg-mem operation
6668 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
6669 %{
6670 instruction_count(3);
6671 dst : S5(write);
6672 src1 : S3(read);
6673 mem : S3(read);
6674 D0 : S0; // big decoder only
6675 DECODE : S1(2); // any decoder for FPU POP
6676 FPU : S4;
6677 MEM : S3; // any mem
6678 %}
6679
6680 // Float mem-reg operation
6681 pipe_class fpu_mem_reg(memory mem, regD src)
6682 %{
6683 instruction_count(2);
6684 src : S5(read);
6685 mem : S3(read);
6686 DECODE : S0; // any decoder for FPU PUSH
6687 D0 : S1; // big decoder only
6688 FPU : S4;
6689 MEM : S3; // any mem
6690 %}
6691
6692 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
6693 %{
6694 instruction_count(3);
6695 src1 : S3(read);
6696 src2 : S3(read);
6697 mem : S3(read);
6698 DECODE : S0(2); // any decoder for FPU PUSH
6699 D0 : S1; // big decoder only
6700 FPU : S4;
6701 MEM : S3; // any mem
6702 %}
6703
6704 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
6705 %{
6706 instruction_count(3);
6707 src1 : S3(read);
6708 src2 : S3(read);
6709 mem : S4(read);
6710 DECODE : S0; // any decoder for FPU PUSH
6711 D0 : S0(2); // big decoder only
6712 FPU : S4;
6713 MEM : S3(2); // any mem
6714 %}
6715
6716 pipe_class fpu_mem_mem(memory dst, memory src1)
6717 %{
6718 instruction_count(2);
6719 src1 : S3(read);
6720 dst : S4(read);
6721 D0 : S0(2); // big decoder only
6722 MEM : S3(2); // any mem
6723 %}
6724
6725 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
6726 %{
6727 instruction_count(3);
6728 src1 : S3(read);
6729 src2 : S3(read);
6730 dst : S4(read);
6731 D0 : S0(3); // big decoder only
6732 FPU : S4;
6733 MEM : S3(3); // any mem
6734 %}
6735
6736 pipe_class fpu_mem_reg_con(memory mem, regD src1)
6737 %{
6738 instruction_count(3);
6739 src1 : S4(read);
6740 mem : S4(read);
6741 DECODE : S0; // any decoder for FPU PUSH
6742 D0 : S0(2); // big decoder only
6743 FPU : S4;
6744 MEM : S3(2); // any mem
6745 %}
6746
6747 // Float load constant
6748 pipe_class fpu_reg_con(regD dst)
6749 %{
6750 instruction_count(2);
6751 dst : S5(write);
6752 D0 : S0; // big decoder only for the load
6753 DECODE : S1; // any decoder for FPU POP
6754 FPU : S4;
6755 MEM : S3; // any mem
6756 %}
6757
6758 // Float load constant
6759 pipe_class fpu_reg_reg_con(regD dst, regD src)
6760 %{
6761 instruction_count(3);
6762 dst : S5(write);
6763 src : S3(read);
6764 D0 : S0; // big decoder only for the load
6765 DECODE : S1(2); // any decoder for FPU POP
6766 FPU : S4;
6767 MEM : S3; // any mem
6768 %}
6769
6770 // UnConditional branch
6771 pipe_class pipe_jmp(label labl)
6772 %{
6773 single_instruction;
6774 BR : S3;
6775 %}
6776
6777 // Conditional branch
6778 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
6779 %{
6780 single_instruction;
6781 cr : S1(read);
6782 BR : S3;
6783 %}
6784
6785 // Allocation idiom
6786 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
6787 %{
6788 instruction_count(1); force_serialization;
6789 fixed_latency(6);
6790 heap_ptr : S3(read);
6791 DECODE : S0(3);
6792 D0 : S2;
6793 MEM : S3;
6794 ALU : S3(2);
6795 dst : S5(write);
6796 BR : S5;
6797 %}
6798
6799 // Generic big/slow expanded idiom
6800 pipe_class pipe_slow()
6801 %{
6802 instruction_count(10); multiple_bundles; force_serialization;
6803 fixed_latency(100);
6804 D0 : S0(2);
6805 MEM : S3(2);
6806 %}
6807
6808 // The real do-nothing guy
6809 pipe_class empty()
6810 %{
6811 instruction_count(0);
6812 %}
6813
6814 // Define the class for the Nop node
6815 define
6816 %{
6817 MachNop = empty;
6818 %}
6819
6820 %}
6821
6822 //----------INSTRUCTIONS-------------------------------------------------------
6823 //
6824 // match -- States which machine-independent subtree may be replaced
6825 // by this instruction.
6826 // ins_cost -- The estimated cost of this instruction is used by instruction
6827 // selection to identify a minimum cost tree of machine
6828 // instructions that matches a tree of machine-independent
6829 // instructions.
6830 // format -- A string providing the disassembly for this instruction.
6831 // The value of an instruction's operand may be inserted
6832 // by referring to it with a '$' prefix.
6833 // opcode -- Three instruction opcodes may be provided. These are referred
6834 // to within an encode class as $primary, $secondary, and $tertiary
6835 // rrspectively. The primary opcode is commonly used to
6836 // indicate the type of machine instruction, while secondary
6837 // and tertiary are often used for prefix options or addressing
6838 // modes.
6839 // ins_encode -- A list of encode classes with parameters. The encode class
6840 // name must have been defined in an 'enc_class' specification
6841 // in the encode section of the architecture description.
6842
6843 // ============================================================================
6844
6845 instruct ShouldNotReachHere() %{
6846 match(Halt);
6847 format %{ "stop\t# ShouldNotReachHere" %}
6848 ins_encode %{
6849 if (is_reachable()) {
6850 const char* str = __ code_string(_halt_reason);
6851 __ stop(str);
6852 }
6853 %}
6854 ins_pipe(pipe_slow);
6855 %}
6856
6857 // ============================================================================
6858
6859 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
6860 // Load Float
6861 instruct MoveF2VL(vlRegF dst, regF src) %{
6862 match(Set dst src);
6863 format %{ "movss $dst,$src\t! load float (4 bytes)" %}
6864 ins_encode %{
6865 ShouldNotReachHere();
6866 %}
6867 ins_pipe( fpu_reg_reg );
6868 %}
6869
6870 // Load Float
6871 instruct MoveF2LEG(legRegF dst, regF src) %{
6872 match(Set dst src);
6873 format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
6874 ins_encode %{
6875 ShouldNotReachHere();
6876 %}
6877 ins_pipe( fpu_reg_reg );
6878 %}
6879
6880 // Load Float
6881 instruct MoveVL2F(regF dst, vlRegF src) %{
6882 match(Set dst src);
6883 format %{ "movss $dst,$src\t! load float (4 bytes)" %}
6884 ins_encode %{
6885 ShouldNotReachHere();
6886 %}
6887 ins_pipe( fpu_reg_reg );
6888 %}
6889
6890 // Load Float
6891 instruct MoveLEG2F(regF dst, legRegF src) %{
6892 match(Set dst src);
6893 format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
6894 ins_encode %{
6895 ShouldNotReachHere();
6896 %}
6897 ins_pipe( fpu_reg_reg );
6898 %}
6899
6900 // Load Double
6901 instruct MoveD2VL(vlRegD dst, regD src) %{
6902 match(Set dst src);
6903 format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
6904 ins_encode %{
6905 ShouldNotReachHere();
6906 %}
6907 ins_pipe( fpu_reg_reg );
6908 %}
6909
6910 // Load Double
6911 instruct MoveD2LEG(legRegD dst, regD src) %{
6912 match(Set dst src);
6913 format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
6914 ins_encode %{
6915 ShouldNotReachHere();
6916 %}
6917 ins_pipe( fpu_reg_reg );
6918 %}
6919
6920 // Load Double
6921 instruct MoveVL2D(regD dst, vlRegD src) %{
6922 match(Set dst src);
6923 format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
6924 ins_encode %{
6925 ShouldNotReachHere();
6926 %}
6927 ins_pipe( fpu_reg_reg );
6928 %}
6929
6930 // Load Double
6931 instruct MoveLEG2D(regD dst, legRegD src) %{
6932 match(Set dst src);
6933 format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
6934 ins_encode %{
6935 ShouldNotReachHere();
6936 %}
6937 ins_pipe( fpu_reg_reg );
6938 %}
6939
6940 //----------Load/Store/Move Instructions---------------------------------------
6941 //----------Load Instructions--------------------------------------------------
6942
6943 // Load Byte (8 bit signed)
6944 instruct loadB(rRegI dst, memory mem)
6945 %{
6946 match(Set dst (LoadB mem));
6947
6948 ins_cost(125);
6949 format %{ "movsbl $dst, $mem\t# byte" %}
6950
6951 ins_encode %{
6952 __ movsbl($dst$$Register, $mem$$Address);
6953 %}
6954
6955 ins_pipe(ialu_reg_mem);
6956 %}
6957
6958 // Load Byte (8 bit signed) into Long Register
6959 instruct loadB2L(rRegL dst, memory mem)
6960 %{
6961 match(Set dst (ConvI2L (LoadB mem)));
6962
6963 ins_cost(125);
6964 format %{ "movsbq $dst, $mem\t# byte -> long" %}
6965
6966 ins_encode %{
6967 __ movsbq($dst$$Register, $mem$$Address);
6968 %}
6969
6970 ins_pipe(ialu_reg_mem);
6971 %}
6972
6973 // Load Unsigned Byte (8 bit UNsigned)
6974 instruct loadUB(rRegI dst, memory mem)
6975 %{
6976 match(Set dst (LoadUB mem));
6977
6978 ins_cost(125);
6979 format %{ "movzbl $dst, $mem\t# ubyte" %}
6980
6981 ins_encode %{
6982 __ movzbl($dst$$Register, $mem$$Address);
6983 %}
6984
6985 ins_pipe(ialu_reg_mem);
6986 %}
6987
6988 // Load Unsigned Byte (8 bit UNsigned) into Long Register
6989 instruct loadUB2L(rRegL dst, memory mem)
6990 %{
6991 match(Set dst (ConvI2L (LoadUB mem)));
6992
6993 ins_cost(125);
6994 format %{ "movzbq $dst, $mem\t# ubyte -> long" %}
6995
6996 ins_encode %{
6997 __ movzbq($dst$$Register, $mem$$Address);
6998 %}
6999
7000 ins_pipe(ialu_reg_mem);
7001 %}
7002
7003 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
7004 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
7005 match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
7006 effect(KILL cr);
7007
7008 format %{ "movzbq $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
7009 "andl $dst, right_n_bits($mask, 8)" %}
7010 ins_encode %{
7011 Register Rdst = $dst$$Register;
7012 __ movzbq(Rdst, $mem$$Address);
7013 __ andl(Rdst, $mask$$constant & right_n_bits(8));
7014 %}
7015 ins_pipe(ialu_reg_mem);
7016 %}
7017
7018 // Load Short (16 bit signed)
7019 instruct loadS(rRegI dst, memory mem)
7020 %{
7021 match(Set dst (LoadS mem));
7022
7023 ins_cost(125);
7024 format %{ "movswl $dst, $mem\t# short" %}
7025
7026 ins_encode %{
7027 __ movswl($dst$$Register, $mem$$Address);
7028 %}
7029
7030 ins_pipe(ialu_reg_mem);
7031 %}
7032
7033 // Load Short (16 bit signed) to Byte (8 bit signed)
7034 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
7035 match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
7036
7037 ins_cost(125);
7038 format %{ "movsbl $dst, $mem\t# short -> byte" %}
7039 ins_encode %{
7040 __ movsbl($dst$$Register, $mem$$Address);
7041 %}
7042 ins_pipe(ialu_reg_mem);
7043 %}
7044
7045 // Load Short (16 bit signed) into Long Register
7046 instruct loadS2L(rRegL dst, memory mem)
7047 %{
7048 match(Set dst (ConvI2L (LoadS mem)));
7049
7050 ins_cost(125);
7051 format %{ "movswq $dst, $mem\t# short -> long" %}
7052
7053 ins_encode %{
7054 __ movswq($dst$$Register, $mem$$Address);
7055 %}
7056
7057 ins_pipe(ialu_reg_mem);
7058 %}
7059
7060 // Load Unsigned Short/Char (16 bit UNsigned)
7061 instruct loadUS(rRegI dst, memory mem)
7062 %{
7063 match(Set dst (LoadUS mem));
7064
7065 ins_cost(125);
7066 format %{ "movzwl $dst, $mem\t# ushort/char" %}
7067
7068 ins_encode %{
7069 __ movzwl($dst$$Register, $mem$$Address);
7070 %}
7071
7072 ins_pipe(ialu_reg_mem);
7073 %}
7074
7075 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
7076 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
7077 match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
7078
7079 ins_cost(125);
7080 format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
7081 ins_encode %{
7082 __ movsbl($dst$$Register, $mem$$Address);
7083 %}
7084 ins_pipe(ialu_reg_mem);
7085 %}
7086
7087 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
7088 instruct loadUS2L(rRegL dst, memory mem)
7089 %{
7090 match(Set dst (ConvI2L (LoadUS mem)));
7091
7092 ins_cost(125);
7093 format %{ "movzwq $dst, $mem\t# ushort/char -> long" %}
7094
7095 ins_encode %{
7096 __ movzwq($dst$$Register, $mem$$Address);
7097 %}
7098
7099 ins_pipe(ialu_reg_mem);
7100 %}
7101
7102 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
7103 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
7104 match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
7105
7106 format %{ "movzbq $dst, $mem\t# ushort/char & 0xFF -> long" %}
7107 ins_encode %{
7108 __ movzbq($dst$$Register, $mem$$Address);
7109 %}
7110 ins_pipe(ialu_reg_mem);
7111 %}
7112
7113 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
7114 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
7115 match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
7116 effect(KILL cr);
7117
7118 format %{ "movzwq $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
7119 "andl $dst, right_n_bits($mask, 16)" %}
7120 ins_encode %{
7121 Register Rdst = $dst$$Register;
7122 __ movzwq(Rdst, $mem$$Address);
7123 __ andl(Rdst, $mask$$constant & right_n_bits(16));
7124 %}
7125 ins_pipe(ialu_reg_mem);
7126 %}
7127
7128 // Load Integer
7129 instruct loadI(rRegI dst, memory mem)
7130 %{
7131 match(Set dst (LoadI mem));
7132
7133 ins_cost(125);
7134 format %{ "movl $dst, $mem\t# int" %}
7135
7136 ins_encode %{
7137 __ movl($dst$$Register, $mem$$Address);
7138 %}
7139
7140 ins_pipe(ialu_reg_mem);
7141 %}
7142
7143 // Load Integer (32 bit signed) to Byte (8 bit signed)
7144 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
7145 match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
7146
7147 ins_cost(125);
7148 format %{ "movsbl $dst, $mem\t# int -> byte" %}
7149 ins_encode %{
7150 __ movsbl($dst$$Register, $mem$$Address);
7151 %}
7152 ins_pipe(ialu_reg_mem);
7153 %}
7154
7155 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
7156 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
7157 match(Set dst (AndI (LoadI mem) mask));
7158
7159 ins_cost(125);
7160 format %{ "movzbl $dst, $mem\t# int -> ubyte" %}
7161 ins_encode %{
7162 __ movzbl($dst$$Register, $mem$$Address);
7163 %}
7164 ins_pipe(ialu_reg_mem);
7165 %}
7166
7167 // Load Integer (32 bit signed) to Short (16 bit signed)
7168 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
7169 match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
7170
7171 ins_cost(125);
7172 format %{ "movswl $dst, $mem\t# int -> short" %}
7173 ins_encode %{
7174 __ movswl($dst$$Register, $mem$$Address);
7175 %}
7176 ins_pipe(ialu_reg_mem);
7177 %}
7178
7179 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
7180 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
7181 match(Set dst (AndI (LoadI mem) mask));
7182
7183 ins_cost(125);
7184 format %{ "movzwl $dst, $mem\t# int -> ushort/char" %}
7185 ins_encode %{
7186 __ movzwl($dst$$Register, $mem$$Address);
7187 %}
7188 ins_pipe(ialu_reg_mem);
7189 %}
7190
7191 // Load Integer into Long Register
7192 instruct loadI2L(rRegL dst, memory mem)
7193 %{
7194 match(Set dst (ConvI2L (LoadI mem)));
7195
7196 ins_cost(125);
7197 format %{ "movslq $dst, $mem\t# int -> long" %}
7198
7199 ins_encode %{
7200 __ movslq($dst$$Register, $mem$$Address);
7201 %}
7202
7203 ins_pipe(ialu_reg_mem);
7204 %}
7205
7206 // Load Integer with mask 0xFF into Long Register
7207 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
7208 match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
7209
7210 format %{ "movzbq $dst, $mem\t# int & 0xFF -> long" %}
7211 ins_encode %{
7212 __ movzbq($dst$$Register, $mem$$Address);
7213 %}
7214 ins_pipe(ialu_reg_mem);
7215 %}
7216
7217 // Load Integer with mask 0xFFFF into Long Register
7218 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
7219 match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
7220
7221 format %{ "movzwq $dst, $mem\t# int & 0xFFFF -> long" %}
7222 ins_encode %{
7223 __ movzwq($dst$$Register, $mem$$Address);
7224 %}
7225 ins_pipe(ialu_reg_mem);
7226 %}
7227
7228 // Load Integer with a 31-bit mask into Long Register
7229 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
7230 match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
7231 effect(KILL cr);
7232
7233 format %{ "movl $dst, $mem\t# int & 31-bit mask -> long\n\t"
7234 "andl $dst, $mask" %}
7235 ins_encode %{
7236 Register Rdst = $dst$$Register;
7237 __ movl(Rdst, $mem$$Address);
7238 __ andl(Rdst, $mask$$constant);
7239 %}
7240 ins_pipe(ialu_reg_mem);
7241 %}
7242
7243 // Load Unsigned Integer into Long Register
7244 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
7245 %{
7246 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7247
7248 ins_cost(125);
7249 format %{ "movl $dst, $mem\t# uint -> long" %}
7250
7251 ins_encode %{
7252 __ movl($dst$$Register, $mem$$Address);
7253 %}
7254
7255 ins_pipe(ialu_reg_mem);
7256 %}
7257
7258 // Load Long
7259 instruct loadL(rRegL dst, memory mem)
7260 %{
7261 match(Set dst (LoadL mem));
7262
7263 ins_cost(125);
7264 format %{ "movq $dst, $mem\t# long" %}
7265
7266 ins_encode %{
7267 __ movq($dst$$Register, $mem$$Address);
7268 %}
7269
7270 ins_pipe(ialu_reg_mem); // XXX
7271 %}
7272
7273 // Load Range
7274 instruct loadRange(rRegI dst, memory mem)
7275 %{
7276 match(Set dst (LoadRange mem));
7277
7278 ins_cost(125); // XXX
7279 format %{ "movl $dst, $mem\t# range" %}
7280 ins_encode %{
7281 __ movl($dst$$Register, $mem$$Address);
7282 %}
7283 ins_pipe(ialu_reg_mem);
7284 %}
7285
7286 // Load Pointer
7287 instruct loadP(rRegP dst, memory mem)
7288 %{
7289 match(Set dst (LoadP mem));
7290 predicate(n->as_Load()->barrier_data() == 0);
7291
7292 ins_cost(125); // XXX
7293 format %{ "movq $dst, $mem\t# ptr" %}
7294 ins_encode %{
7295 __ movq($dst$$Register, $mem$$Address);
7296 %}
7297 ins_pipe(ialu_reg_mem); // XXX
7298 %}
7299
7300 // Load Compressed Pointer
7301 instruct loadN(rRegN dst, memory mem)
7302 %{
7303 predicate(n->as_Load()->barrier_data() == 0);
7304 match(Set dst (LoadN mem));
7305
7306 ins_cost(125); // XXX
7307 format %{ "movl $dst, $mem\t# compressed ptr" %}
7308 ins_encode %{
7309 __ movl($dst$$Register, $mem$$Address);
7310 %}
7311 ins_pipe(ialu_reg_mem); // XXX
7312 %}
7313
7314
7315 // Load Klass Pointer
7316 instruct loadKlass(rRegP dst, memory mem)
7317 %{
7318 match(Set dst (LoadKlass mem));
7319
7320 ins_cost(125); // XXX
7321 format %{ "movq $dst, $mem\t# class" %}
7322 ins_encode %{
7323 __ movq($dst$$Register, $mem$$Address);
7324 %}
7325 ins_pipe(ialu_reg_mem); // XXX
7326 %}
7327
7328 // Load narrow Klass Pointer
7329 instruct loadNKlass(rRegN dst, memory mem)
7330 %{
7331 predicate(!UseCompactObjectHeaders);
7332 match(Set dst (LoadNKlass mem));
7333
7334 ins_cost(125); // XXX
7335 format %{ "movl $dst, $mem\t# compressed klass ptr" %}
7336 ins_encode %{
7337 __ movl($dst$$Register, $mem$$Address);
7338 %}
7339 ins_pipe(ialu_reg_mem); // XXX
7340 %}
7341
7342 instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr)
7343 %{
7344 predicate(UseCompactObjectHeaders);
7345 match(Set dst (LoadNKlass mem));
7346 effect(KILL cr);
7347 ins_cost(125);
7348 format %{
7349 "movl $dst, $mem\t# compressed klass ptr, shifted\n\t"
7350 "shrl $dst, markWord::klass_shift_at_offset"
7351 %}
7352 ins_encode %{
7353 __ movl($dst$$Register, $mem$$Address);
7354 __ shrl($dst$$Register, markWord::klass_shift_at_offset);
7355 %}
7356 ins_pipe(ialu_reg_mem);
7357 %}
7358
7359 // Load Float
7360 instruct loadF(regF dst, memory mem)
7361 %{
7362 match(Set dst (LoadF mem));
7363
7364 ins_cost(145); // XXX
7365 format %{ "movss $dst, $mem\t# float" %}
7366 ins_encode %{
7367 __ movflt($dst$$XMMRegister, $mem$$Address);
7368 %}
7369 ins_pipe(pipe_slow); // XXX
7370 %}
7371
7372 // Load Double
7373 instruct loadD_partial(regD dst, memory mem)
7374 %{
7375 predicate(!UseXmmLoadAndClearUpper);
7376 match(Set dst (LoadD mem));
7377
7378 ins_cost(145); // XXX
7379 format %{ "movlpd $dst, $mem\t# double" %}
7380 ins_encode %{
7381 __ movdbl($dst$$XMMRegister, $mem$$Address);
7382 %}
7383 ins_pipe(pipe_slow); // XXX
7384 %}
7385
7386 instruct loadD(regD dst, memory mem)
7387 %{
7388 predicate(UseXmmLoadAndClearUpper);
7389 match(Set dst (LoadD mem));
7390
7391 ins_cost(145); // XXX
7392 format %{ "movsd $dst, $mem\t# double" %}
7393 ins_encode %{
7394 __ movdbl($dst$$XMMRegister, $mem$$Address);
7395 %}
7396 ins_pipe(pipe_slow); // XXX
7397 %}
7398
7399 instruct loadAOTRCAddress(rRegP dst, immAOTRuntimeConstantsAddress con)
7400 %{
7401 match(Set dst con);
7402
7403 format %{ "leaq $dst, $con\t# AOT Runtime Constants Address" %}
7404
7405 ins_encode %{
7406 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
7407 %}
7408
7409 ins_pipe(ialu_reg_fat);
7410 %}
7411
7412 // min = java.lang.Math.min(float a, float b)
7413 // max = java.lang.Math.max(float a, float b)
7414 instruct minmaxF_reg_avx10_2(regF dst, regF a, regF b)
7415 %{
7416 predicate(VM_Version::supports_avx10_2() && !VLoopReductions::is_reduction(n));
7417 match(Set dst (MaxF a b));
7418 match(Set dst (MinF a b));
7419
7420 format %{ "minmaxF $dst, $a, $b" %}
7421 ins_encode %{
7422 int opcode = this->ideal_Opcode();
7423 __ sminmax_fp_avx10_2(opcode, T_FLOAT, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister);
7424 %}
7425 ins_pipe( pipe_slow );
7426 %}
7427
7428 instruct minmaxF_reduction_reg_avx10_2(regF dst, regF a, regF b, rRegI rtmp, rFlagsReg cr)
7429 %{
7430 predicate(VM_Version::supports_avx10_2() && VLoopReductions::is_reduction(n));
7431 match(Set dst (MaxF a b));
7432 match(Set dst (MinF a b));
7433 effect(USE a, USE b, TEMP rtmp, KILL cr);
7434
7435 format %{ "minmaxF_reduction $dst, $a, $b \t! using $rtmp as TEMP" %}
7436 ins_encode %{
7437 int opcode = this->ideal_Opcode();
7438 bool min = (opcode == Op_MinF) ? true : false;
7439 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7440 min, fp_prec_flt /*pt*/);
7441 %}
7442 ins_pipe( pipe_slow );
7443 %}
7444
7445 // min = java.lang.Math.min(float a, float b)
7446 // max = java.lang.Math.max(float a, float b)
7447 instruct minmaxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp)
7448 %{
7449 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
7450 match(Set dst (MaxF a b));
7451 match(Set dst (MinF a b));
7452 effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
7453
7454 format %{ "minmaxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
7455 ins_encode %{
7456 int opcode = this->ideal_Opcode();
7457 int param_opcode = (opcode == Op_MinF) ? Op_MinV : Op_MaxV;
7458 __ vminmax_fp(param_opcode, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister,
7459 $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
7460 %}
7461 ins_pipe( pipe_slow );
7462 %}
7463
7464 instruct minmaxF_reduction_reg(legRegF dst, legRegF a, legRegF b, rRegI rtmp, rFlagsReg cr)
7465 %{
7466 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
7467 match(Set dst (MaxF a b));
7468 match(Set dst (MinF a b));
7469 effect(USE a, USE b, TEMP rtmp, KILL cr);
7470
7471 format %{ "minmaxF_reduction $dst, $a, $b \t!using $rtmp as TEMP" %}
7472 ins_encode %{
7473 int opcode = this->ideal_Opcode();
7474 bool min = (opcode == Op_MinF) ? true : false;
7475 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7476 min, fp_prec_flt /*pt*/);
7477 %}
7478 ins_pipe( pipe_slow );
7479 %}
7480
7481 // min = java.lang.Math.min(double a, double b)
7482 // max = java.lang.Math.max(double a, double b)
7483 instruct minmaxD_reg_avx10_2(regD dst, regD a, regD b)
7484 %{
7485 predicate(VM_Version::supports_avx10_2() && !VLoopReductions::is_reduction(n));
7486 match(Set dst (MaxD a b));
7487 match(Set dst (MinD a b));
7488
7489 format %{ "minmaxD $dst, $a, $b" %}
7490 ins_encode %{
7491 int opcode = this->ideal_Opcode();
7492 __ sminmax_fp_avx10_2(opcode, T_DOUBLE, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister);
7493 %}
7494 ins_pipe( pipe_slow );
7495 %}
7496
7497 instruct minmaxD_reduction_reg_avx10_2(regD dst, regD a, regD b, rRegI rtmp, rFlagsReg cr)
7498 %{
7499 predicate(VM_Version::supports_avx10_2() && VLoopReductions::is_reduction(n));
7500 match(Set dst (MaxD a b));
7501 match(Set dst (MinD a b));
7502 effect(USE a, USE b, TEMP rtmp, KILL cr);
7503
7504 format %{ "minmaxD_reduction $dst, $a, $b \t! using $rtmp as TEMP" %}
7505 ins_encode %{
7506 int opcode = this->ideal_Opcode();
7507 bool min = (opcode == Op_MinD) ? true : false;
7508 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7509 min, fp_prec_dbl /*pt*/);
7510 %}
7511 ins_pipe( pipe_slow );
7512 %}
7513
7514 // min = java.lang.Math.min(double a, double b)
7515 // max = java.lang.Math.max(double a, double b)
7516 instruct minmaxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp)
7517 %{
7518 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
7519 match(Set dst (MaxD a b));
7520 match(Set dst (MinD a b));
7521 effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
7522
7523 format %{ "minmaxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
7524 ins_encode %{
7525 int opcode = this->ideal_Opcode();
7526 int param_opcode = (opcode == Op_MinD) ? Op_MinV : Op_MaxV;
7527 __ vminmax_fp(param_opcode, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister,
7528 $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
7529 %}
7530 ins_pipe( pipe_slow );
7531 %}
7532
7533 instruct minmaxD_reduction_reg(legRegD dst, legRegD a, legRegD b, rRegL rtmp, rFlagsReg cr)
7534 %{
7535 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
7536 match(Set dst (MaxD a b));
7537 match(Set dst (MinD a b));
7538 effect(USE a, USE b, TEMP rtmp, KILL cr);
7539
7540 format %{ "minmaxD_reduction $dst, $a, $b \t! using $rtmp as TEMP" %}
7541 ins_encode %{
7542 int opcode = this->ideal_Opcode();
7543 bool min = (opcode == Op_MinD) ? true : false;
7544 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7545 min, fp_prec_dbl /*pt*/);
7546 %}
7547 ins_pipe( pipe_slow );
7548 %}
7549
7550 // Load Effective Address
7551 instruct leaP8(rRegP dst, indOffset8 mem)
7552 %{
7553 match(Set dst mem);
7554
7555 ins_cost(110); // XXX
7556 format %{ "leaq $dst, $mem\t# ptr 8" %}
7557 ins_encode %{
7558 __ leaq($dst$$Register, $mem$$Address);
7559 %}
7560 ins_pipe(ialu_reg_reg_fat);
7561 %}
7562
7563 instruct leaP32(rRegP dst, indOffset32 mem)
7564 %{
7565 match(Set dst mem);
7566
7567 ins_cost(110);
7568 format %{ "leaq $dst, $mem\t# ptr 32" %}
7569 ins_encode %{
7570 __ leaq($dst$$Register, $mem$$Address);
7571 %}
7572 ins_pipe(ialu_reg_reg_fat);
7573 %}
7574
7575 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
7576 %{
7577 match(Set dst mem);
7578
7579 ins_cost(110);
7580 format %{ "leaq $dst, $mem\t# ptr idxoff" %}
7581 ins_encode %{
7582 __ leaq($dst$$Register, $mem$$Address);
7583 %}
7584 ins_pipe(ialu_reg_reg_fat);
7585 %}
7586
7587 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
7588 %{
7589 match(Set dst mem);
7590
7591 ins_cost(110);
7592 format %{ "leaq $dst, $mem\t# ptr idxscale" %}
7593 ins_encode %{
7594 __ leaq($dst$$Register, $mem$$Address);
7595 %}
7596 ins_pipe(ialu_reg_reg_fat);
7597 %}
7598
7599 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
7600 %{
7601 match(Set dst mem);
7602
7603 ins_cost(110);
7604 format %{ "leaq $dst, $mem\t# ptr idxscale" %}
7605 ins_encode %{
7606 __ leaq($dst$$Register, $mem$$Address);
7607 %}
7608 ins_pipe(ialu_reg_reg_fat);
7609 %}
7610
7611 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
7612 %{
7613 match(Set dst mem);
7614
7615 ins_cost(110);
7616 format %{ "leaq $dst, $mem\t# ptr idxscaleoff" %}
7617 ins_encode %{
7618 __ leaq($dst$$Register, $mem$$Address);
7619 %}
7620 ins_pipe(ialu_reg_reg_fat);
7621 %}
7622
7623 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
7624 %{
7625 match(Set dst mem);
7626
7627 ins_cost(110);
7628 format %{ "leaq $dst, $mem\t# ptr posidxoff" %}
7629 ins_encode %{
7630 __ leaq($dst$$Register, $mem$$Address);
7631 %}
7632 ins_pipe(ialu_reg_reg_fat);
7633 %}
7634
7635 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
7636 %{
7637 match(Set dst mem);
7638
7639 ins_cost(110);
7640 format %{ "leaq $dst, $mem\t# ptr posidxscaleoff" %}
7641 ins_encode %{
7642 __ leaq($dst$$Register, $mem$$Address);
7643 %}
7644 ins_pipe(ialu_reg_reg_fat);
7645 %}
7646
7647 // Load Effective Address which uses Narrow (32-bits) oop
7648 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
7649 %{
7650 predicate(UseCompressedOops && (CompressedOops::shift() != 0));
7651 match(Set dst mem);
7652
7653 ins_cost(110);
7654 format %{ "leaq $dst, $mem\t# ptr compressedoopoff32" %}
7655 ins_encode %{
7656 __ leaq($dst$$Register, $mem$$Address);
7657 %}
7658 ins_pipe(ialu_reg_reg_fat);
7659 %}
7660
7661 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
7662 %{
7663 predicate(CompressedOops::shift() == 0);
7664 match(Set dst mem);
7665
7666 ins_cost(110); // XXX
7667 format %{ "leaq $dst, $mem\t# ptr off8narrow" %}
7668 ins_encode %{
7669 __ leaq($dst$$Register, $mem$$Address);
7670 %}
7671 ins_pipe(ialu_reg_reg_fat);
7672 %}
7673
7674 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
7675 %{
7676 predicate(CompressedOops::shift() == 0);
7677 match(Set dst mem);
7678
7679 ins_cost(110);
7680 format %{ "leaq $dst, $mem\t# ptr off32narrow" %}
7681 ins_encode %{
7682 __ leaq($dst$$Register, $mem$$Address);
7683 %}
7684 ins_pipe(ialu_reg_reg_fat);
7685 %}
7686
7687 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
7688 %{
7689 predicate(CompressedOops::shift() == 0);
7690 match(Set dst mem);
7691
7692 ins_cost(110);
7693 format %{ "leaq $dst, $mem\t# ptr idxoffnarrow" %}
7694 ins_encode %{
7695 __ leaq($dst$$Register, $mem$$Address);
7696 %}
7697 ins_pipe(ialu_reg_reg_fat);
7698 %}
7699
7700 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
7701 %{
7702 predicate(CompressedOops::shift() == 0);
7703 match(Set dst mem);
7704
7705 ins_cost(110);
7706 format %{ "leaq $dst, $mem\t# ptr idxscalenarrow" %}
7707 ins_encode %{
7708 __ leaq($dst$$Register, $mem$$Address);
7709 %}
7710 ins_pipe(ialu_reg_reg_fat);
7711 %}
7712
7713 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
7714 %{
7715 predicate(CompressedOops::shift() == 0);
7716 match(Set dst mem);
7717
7718 ins_cost(110);
7719 format %{ "leaq $dst, $mem\t# ptr idxscaleoffnarrow" %}
7720 ins_encode %{
7721 __ leaq($dst$$Register, $mem$$Address);
7722 %}
7723 ins_pipe(ialu_reg_reg_fat);
7724 %}
7725
7726 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
7727 %{
7728 predicate(CompressedOops::shift() == 0);
7729 match(Set dst mem);
7730
7731 ins_cost(110);
7732 format %{ "leaq $dst, $mem\t# ptr posidxoffnarrow" %}
7733 ins_encode %{
7734 __ leaq($dst$$Register, $mem$$Address);
7735 %}
7736 ins_pipe(ialu_reg_reg_fat);
7737 %}
7738
7739 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
7740 %{
7741 predicate(CompressedOops::shift() == 0);
7742 match(Set dst mem);
7743
7744 ins_cost(110);
7745 format %{ "leaq $dst, $mem\t# ptr posidxscaleoffnarrow" %}
7746 ins_encode %{
7747 __ leaq($dst$$Register, $mem$$Address);
7748 %}
7749 ins_pipe(ialu_reg_reg_fat);
7750 %}
7751
7752 instruct loadConI(rRegI dst, immI src)
7753 %{
7754 match(Set dst src);
7755
7756 format %{ "movl $dst, $src\t# int" %}
7757 ins_encode %{
7758 __ movl($dst$$Register, $src$$constant);
7759 %}
7760 ins_pipe(ialu_reg_fat); // XXX
7761 %}
7762
7763 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
7764 %{
7765 match(Set dst src);
7766 effect(KILL cr);
7767
7768 ins_cost(50);
7769 format %{ "xorl $dst, $dst\t# int" %}
7770 ins_encode %{
7771 __ xorl($dst$$Register, $dst$$Register);
7772 %}
7773 ins_pipe(ialu_reg);
7774 %}
7775
7776 instruct loadConL(rRegL dst, immL src)
7777 %{
7778 match(Set dst src);
7779
7780 ins_cost(150);
7781 format %{ "movq $dst, $src\t# long" %}
7782 ins_encode %{
7783 __ mov64($dst$$Register, $src$$constant);
7784 %}
7785 ins_pipe(ialu_reg);
7786 %}
7787
7788 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
7789 %{
7790 match(Set dst src);
7791 effect(KILL cr);
7792
7793 ins_cost(50);
7794 format %{ "xorl $dst, $dst\t# long" %}
7795 ins_encode %{
7796 __ xorl($dst$$Register, $dst$$Register);
7797 %}
7798 ins_pipe(ialu_reg); // XXX
7799 %}
7800
7801 instruct loadConUL32(rRegL dst, immUL32 src)
7802 %{
7803 match(Set dst src);
7804
7805 ins_cost(60);
7806 format %{ "movl $dst, $src\t# long (unsigned 32-bit)" %}
7807 ins_encode %{
7808 __ movl($dst$$Register, $src$$constant);
7809 %}
7810 ins_pipe(ialu_reg);
7811 %}
7812
7813 instruct loadConL32(rRegL dst, immL32 src)
7814 %{
7815 match(Set dst src);
7816
7817 ins_cost(70);
7818 format %{ "movq $dst, $src\t# long (32-bit)" %}
7819 ins_encode %{
7820 __ movq($dst$$Register, $src$$constant);
7821 %}
7822 ins_pipe(ialu_reg);
7823 %}
7824
7825 instruct loadConP(rRegP dst, immP con) %{
7826 match(Set dst con);
7827
7828 format %{ "movq $dst, $con\t# ptr" %}
7829 ins_encode %{
7830 __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
7831 %}
7832 ins_pipe(ialu_reg_fat); // XXX
7833 %}
7834
7835 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
7836 %{
7837 match(Set dst src);
7838 effect(KILL cr);
7839
7840 ins_cost(50);
7841 format %{ "xorl $dst, $dst\t# ptr" %}
7842 ins_encode %{
7843 __ xorl($dst$$Register, $dst$$Register);
7844 %}
7845 ins_pipe(ialu_reg);
7846 %}
7847
7848 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
7849 %{
7850 match(Set dst src);
7851 effect(KILL cr);
7852
7853 ins_cost(60);
7854 format %{ "movl $dst, $src\t# ptr (positive 32-bit)" %}
7855 ins_encode %{
7856 __ movl($dst$$Register, $src$$constant);
7857 %}
7858 ins_pipe(ialu_reg);
7859 %}
7860
7861 instruct loadConF(regF dst, immF con) %{
7862 match(Set dst con);
7863 ins_cost(125);
7864 format %{ "movss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
7865 ins_encode %{
7866 __ movflt($dst$$XMMRegister, $constantaddress($con));
7867 %}
7868 ins_pipe(pipe_slow);
7869 %}
7870
7871 instruct loadConH(regF dst, immH con) %{
7872 match(Set dst con);
7873 ins_cost(125);
7874 format %{ "movss $dst, [$constantaddress]\t# load from constant table: halffloat=$con" %}
7875 ins_encode %{
7876 __ movflt($dst$$XMMRegister, $constantaddress($con));
7877 %}
7878 ins_pipe(pipe_slow);
7879 %}
7880
7881 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
7882 match(Set dst src);
7883 effect(KILL cr);
7884 format %{ "xorq $dst, $src\t# compressed null pointer" %}
7885 ins_encode %{
7886 __ xorq($dst$$Register, $dst$$Register);
7887 %}
7888 ins_pipe(ialu_reg);
7889 %}
7890
7891 instruct loadConN(rRegN dst, immN src) %{
7892 match(Set dst src);
7893
7894 ins_cost(125);
7895 format %{ "movl $dst, $src\t# compressed ptr" %}
7896 ins_encode %{
7897 address con = (address)$src$$constant;
7898 if (con == nullptr) {
7899 ShouldNotReachHere();
7900 } else {
7901 __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
7902 }
7903 %}
7904 ins_pipe(ialu_reg_fat); // XXX
7905 %}
7906
7907 instruct loadConNKlass(rRegN dst, immNKlass src) %{
7908 match(Set dst src);
7909
7910 ins_cost(125);
7911 format %{ "movl $dst, $src\t# compressed klass ptr" %}
7912 ins_encode %{
7913 address con = (address)$src$$constant;
7914 if (con == nullptr) {
7915 ShouldNotReachHere();
7916 } else {
7917 __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
7918 }
7919 %}
7920 ins_pipe(ialu_reg_fat); // XXX
7921 %}
7922
7923 instruct loadConF0(regF dst, immF0 src)
7924 %{
7925 match(Set dst src);
7926 ins_cost(100);
7927
7928 format %{ "xorps $dst, $dst\t# float 0.0" %}
7929 ins_encode %{
7930 __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
7931 %}
7932 ins_pipe(pipe_slow);
7933 %}
7934
7935 // Use the same format since predicate() can not be used here.
7936 instruct loadConD(regD dst, immD con) %{
7937 match(Set dst con);
7938 ins_cost(125);
7939 format %{ "movsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
7940 ins_encode %{
7941 __ movdbl($dst$$XMMRegister, $constantaddress($con));
7942 %}
7943 ins_pipe(pipe_slow);
7944 %}
7945
7946 instruct loadConD0(regD dst, immD0 src)
7947 %{
7948 match(Set dst src);
7949 ins_cost(100);
7950
7951 format %{ "xorpd $dst, $dst\t# double 0.0" %}
7952 ins_encode %{
7953 __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
7954 %}
7955 ins_pipe(pipe_slow);
7956 %}
7957
7958 instruct loadSSI(rRegI dst, stackSlotI src)
7959 %{
7960 match(Set dst src);
7961
7962 ins_cost(125);
7963 format %{ "movl $dst, $src\t# int stk" %}
7964 ins_encode %{
7965 __ movl($dst$$Register, $src$$Address);
7966 %}
7967 ins_pipe(ialu_reg_mem);
7968 %}
7969
7970 instruct loadSSL(rRegL dst, stackSlotL src)
7971 %{
7972 match(Set dst src);
7973
7974 ins_cost(125);
7975 format %{ "movq $dst, $src\t# long stk" %}
7976 ins_encode %{
7977 __ movq($dst$$Register, $src$$Address);
7978 %}
7979 ins_pipe(ialu_reg_mem);
7980 %}
7981
7982 instruct loadSSP(rRegP dst, stackSlotP src)
7983 %{
7984 match(Set dst src);
7985
7986 ins_cost(125);
7987 format %{ "movq $dst, $src\t# ptr stk" %}
7988 ins_encode %{
7989 __ movq($dst$$Register, $src$$Address);
7990 %}
7991 ins_pipe(ialu_reg_mem);
7992 %}
7993
7994 instruct loadSSF(regF dst, stackSlotF src)
7995 %{
7996 match(Set dst src);
7997
7998 ins_cost(125);
7999 format %{ "movss $dst, $src\t# float stk" %}
8000 ins_encode %{
8001 __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
8002 %}
8003 ins_pipe(pipe_slow); // XXX
8004 %}
8005
8006 // Use the same format since predicate() can not be used here.
8007 instruct loadSSD(regD dst, stackSlotD src)
8008 %{
8009 match(Set dst src);
8010
8011 ins_cost(125);
8012 format %{ "movsd $dst, $src\t# double stk" %}
8013 ins_encode %{
8014 __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
8015 %}
8016 ins_pipe(pipe_slow); // XXX
8017 %}
8018
8019 // Prefetch instructions for allocation.
8020 // Must be safe to execute with invalid address (cannot fault).
8021
8022 instruct prefetchAlloc( memory mem ) %{
8023 predicate(AllocatePrefetchInstr==3);
8024 match(PrefetchAllocation mem);
8025 ins_cost(125);
8026
8027 format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
8028 ins_encode %{
8029 __ prefetchw($mem$$Address);
8030 %}
8031 ins_pipe(ialu_mem);
8032 %}
8033
8034 instruct prefetchAllocNTA( memory mem ) %{
8035 predicate(AllocatePrefetchInstr==0);
8036 match(PrefetchAllocation mem);
8037 ins_cost(125);
8038
8039 format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
8040 ins_encode %{
8041 __ prefetchnta($mem$$Address);
8042 %}
8043 ins_pipe(ialu_mem);
8044 %}
8045
8046 instruct prefetchAllocT0( memory mem ) %{
8047 predicate(AllocatePrefetchInstr==1);
8048 match(PrefetchAllocation mem);
8049 ins_cost(125);
8050
8051 format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
8052 ins_encode %{
8053 __ prefetcht0($mem$$Address);
8054 %}
8055 ins_pipe(ialu_mem);
8056 %}
8057
8058 instruct prefetchAllocT2( memory mem ) %{
8059 predicate(AllocatePrefetchInstr==2);
8060 match(PrefetchAllocation mem);
8061 ins_cost(125);
8062
8063 format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
8064 ins_encode %{
8065 __ prefetcht2($mem$$Address);
8066 %}
8067 ins_pipe(ialu_mem);
8068 %}
8069
8070 //----------Store Instructions-------------------------------------------------
8071
8072 // Store Byte
8073 instruct storeB(memory mem, rRegI src)
8074 %{
8075 match(Set mem (StoreB mem src));
8076
8077 ins_cost(125); // XXX
8078 format %{ "movb $mem, $src\t# byte" %}
8079 ins_encode %{
8080 __ movb($mem$$Address, $src$$Register);
8081 %}
8082 ins_pipe(ialu_mem_reg);
8083 %}
8084
8085 // Store Char/Short
8086 instruct storeC(memory mem, rRegI src)
8087 %{
8088 match(Set mem (StoreC mem src));
8089
8090 ins_cost(125); // XXX
8091 format %{ "movw $mem, $src\t# char/short" %}
8092 ins_encode %{
8093 __ movw($mem$$Address, $src$$Register);
8094 %}
8095 ins_pipe(ialu_mem_reg);
8096 %}
8097
8098 // Store Integer
8099 instruct storeI(memory mem, rRegI src)
8100 %{
8101 match(Set mem (StoreI mem src));
8102
8103 ins_cost(125); // XXX
8104 format %{ "movl $mem, $src\t# int" %}
8105 ins_encode %{
8106 __ movl($mem$$Address, $src$$Register);
8107 %}
8108 ins_pipe(ialu_mem_reg);
8109 %}
8110
8111 // Store Long
8112 instruct storeL(memory mem, rRegL src)
8113 %{
8114 match(Set mem (StoreL mem src));
8115
8116 ins_cost(125); // XXX
8117 format %{ "movq $mem, $src\t# long" %}
8118 ins_encode %{
8119 __ movq($mem$$Address, $src$$Register);
8120 %}
8121 ins_pipe(ialu_mem_reg); // XXX
8122 %}
8123
8124 // Store Pointer
8125 instruct storeP(memory mem, any_RegP src)
8126 %{
8127 predicate(n->as_Store()->barrier_data() == 0);
8128 match(Set mem (StoreP mem src));
8129
8130 ins_cost(125); // XXX
8131 format %{ "movq $mem, $src\t# ptr" %}
8132 ins_encode %{
8133 __ movq($mem$$Address, $src$$Register);
8134 %}
8135 ins_pipe(ialu_mem_reg);
8136 %}
8137
8138 instruct storeImmP0(memory mem, immP0 zero)
8139 %{
8140 predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
8141 match(Set mem (StoreP mem zero));
8142
8143 ins_cost(125); // XXX
8144 format %{ "movq $mem, R12\t# ptr (R12_heapbase==0)" %}
8145 ins_encode %{
8146 __ movq($mem$$Address, r12);
8147 %}
8148 ins_pipe(ialu_mem_reg);
8149 %}
8150
8151 // Store Null Pointer, mark word, or other simple pointer constant.
8152 instruct storeImmP(memory mem, immP31 src)
8153 %{
8154 predicate(n->as_Store()->barrier_data() == 0);
8155 match(Set mem (StoreP mem src));
8156
8157 ins_cost(150); // XXX
8158 format %{ "movq $mem, $src\t# ptr" %}
8159 ins_encode %{
8160 __ movq($mem$$Address, $src$$constant);
8161 %}
8162 ins_pipe(ialu_mem_imm);
8163 %}
8164
8165 // Store Compressed Pointer
8166 instruct storeN(memory mem, rRegN src)
8167 %{
8168 predicate(n->as_Store()->barrier_data() == 0);
8169 match(Set mem (StoreN mem src));
8170
8171 ins_cost(125); // XXX
8172 format %{ "movl $mem, $src\t# compressed ptr" %}
8173 ins_encode %{
8174 __ movl($mem$$Address, $src$$Register);
8175 %}
8176 ins_pipe(ialu_mem_reg);
8177 %}
8178
8179 instruct storeNKlass(memory mem, rRegN src)
8180 %{
8181 match(Set mem (StoreNKlass mem src));
8182
8183 ins_cost(125); // XXX
8184 format %{ "movl $mem, $src\t# compressed klass ptr" %}
8185 ins_encode %{
8186 __ movl($mem$$Address, $src$$Register);
8187 %}
8188 ins_pipe(ialu_mem_reg);
8189 %}
8190
8191 instruct storeImmN0(memory mem, immN0 zero)
8192 %{
8193 predicate(CompressedOops::base() == nullptr && n->as_Store()->barrier_data() == 0);
8194 match(Set mem (StoreN mem zero));
8195
8196 ins_cost(125); // XXX
8197 format %{ "movl $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
8198 ins_encode %{
8199 __ movl($mem$$Address, r12);
8200 %}
8201 ins_pipe(ialu_mem_reg);
8202 %}
8203
8204 instruct storeImmN(memory mem, immN src)
8205 %{
8206 predicate(n->as_Store()->barrier_data() == 0);
8207 match(Set mem (StoreN mem src));
8208
8209 ins_cost(150); // XXX
8210 format %{ "movl $mem, $src\t# compressed ptr" %}
8211 ins_encode %{
8212 address con = (address)$src$$constant;
8213 if (con == nullptr) {
8214 __ movl($mem$$Address, 0);
8215 } else {
8216 __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
8217 }
8218 %}
8219 ins_pipe(ialu_mem_imm);
8220 %}
8221
8222 instruct storeImmNKlass(memory mem, immNKlass src)
8223 %{
8224 match(Set mem (StoreNKlass mem src));
8225
8226 ins_cost(150); // XXX
8227 format %{ "movl $mem, $src\t# compressed klass ptr" %}
8228 ins_encode %{
8229 __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
8230 %}
8231 ins_pipe(ialu_mem_imm);
8232 %}
8233
8234 // Store Integer Immediate
8235 instruct storeImmI0(memory mem, immI_0 zero)
8236 %{
8237 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8238 match(Set mem (StoreI mem zero));
8239
8240 ins_cost(125); // XXX
8241 format %{ "movl $mem, R12\t# int (R12_heapbase==0)" %}
8242 ins_encode %{
8243 __ movl($mem$$Address, r12);
8244 %}
8245 ins_pipe(ialu_mem_reg);
8246 %}
8247
8248 instruct storeImmI(memory mem, immI src)
8249 %{
8250 match(Set mem (StoreI mem src));
8251
8252 ins_cost(150);
8253 format %{ "movl $mem, $src\t# int" %}
8254 ins_encode %{
8255 __ movl($mem$$Address, $src$$constant);
8256 %}
8257 ins_pipe(ialu_mem_imm);
8258 %}
8259
8260 // Store Long Immediate
8261 instruct storeImmL0(memory mem, immL0 zero)
8262 %{
8263 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8264 match(Set mem (StoreL mem zero));
8265
8266 ins_cost(125); // XXX
8267 format %{ "movq $mem, R12\t# long (R12_heapbase==0)" %}
8268 ins_encode %{
8269 __ movq($mem$$Address, r12);
8270 %}
8271 ins_pipe(ialu_mem_reg);
8272 %}
8273
8274 instruct storeImmL(memory mem, immL32 src)
8275 %{
8276 match(Set mem (StoreL mem src));
8277
8278 ins_cost(150);
8279 format %{ "movq $mem, $src\t# long" %}
8280 ins_encode %{
8281 __ movq($mem$$Address, $src$$constant);
8282 %}
8283 ins_pipe(ialu_mem_imm);
8284 %}
8285
8286 // Store Short/Char Immediate
8287 instruct storeImmC0(memory mem, immI_0 zero)
8288 %{
8289 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8290 match(Set mem (StoreC mem zero));
8291
8292 ins_cost(125); // XXX
8293 format %{ "movw $mem, R12\t# short/char (R12_heapbase==0)" %}
8294 ins_encode %{
8295 __ movw($mem$$Address, r12);
8296 %}
8297 ins_pipe(ialu_mem_reg);
8298 %}
8299
8300 instruct storeImmI16(memory mem, immI16 src)
8301 %{
8302 predicate(UseStoreImmI16);
8303 match(Set mem (StoreC mem src));
8304
8305 ins_cost(150);
8306 format %{ "movw $mem, $src\t# short/char" %}
8307 ins_encode %{
8308 __ movw($mem$$Address, $src$$constant);
8309 %}
8310 ins_pipe(ialu_mem_imm);
8311 %}
8312
8313 // Store Byte Immediate
8314 instruct storeImmB0(memory mem, immI_0 zero)
8315 %{
8316 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8317 match(Set mem (StoreB mem zero));
8318
8319 ins_cost(125); // XXX
8320 format %{ "movb $mem, R12\t# short/char (R12_heapbase==0)" %}
8321 ins_encode %{
8322 __ movb($mem$$Address, r12);
8323 %}
8324 ins_pipe(ialu_mem_reg);
8325 %}
8326
8327 instruct storeImmB(memory mem, immI8 src)
8328 %{
8329 match(Set mem (StoreB mem src));
8330
8331 ins_cost(150); // XXX
8332 format %{ "movb $mem, $src\t# byte" %}
8333 ins_encode %{
8334 __ movb($mem$$Address, $src$$constant);
8335 %}
8336 ins_pipe(ialu_mem_imm);
8337 %}
8338
8339 // Store Float
8340 instruct storeF(memory mem, regF src)
8341 %{
8342 match(Set mem (StoreF mem src));
8343
8344 ins_cost(95); // XXX
8345 format %{ "movss $mem, $src\t# float" %}
8346 ins_encode %{
8347 __ movflt($mem$$Address, $src$$XMMRegister);
8348 %}
8349 ins_pipe(pipe_slow); // XXX
8350 %}
8351
8352 // Store immediate Float value (it is faster than store from XMM register)
8353 instruct storeF0(memory mem, immF0 zero)
8354 %{
8355 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8356 match(Set mem (StoreF mem zero));
8357
8358 ins_cost(25); // XXX
8359 format %{ "movl $mem, R12\t# float 0. (R12_heapbase==0)" %}
8360 ins_encode %{
8361 __ movl($mem$$Address, r12);
8362 %}
8363 ins_pipe(ialu_mem_reg);
8364 %}
8365
8366 instruct storeF_imm(memory mem, immF src)
8367 %{
8368 match(Set mem (StoreF mem src));
8369
8370 ins_cost(50);
8371 format %{ "movl $mem, $src\t# float" %}
8372 ins_encode %{
8373 __ movl($mem$$Address, jint_cast($src$$constant));
8374 %}
8375 ins_pipe(ialu_mem_imm);
8376 %}
8377
8378 // Store Double
8379 instruct storeD(memory mem, regD src)
8380 %{
8381 match(Set mem (StoreD mem src));
8382
8383 ins_cost(95); // XXX
8384 format %{ "movsd $mem, $src\t# double" %}
8385 ins_encode %{
8386 __ movdbl($mem$$Address, $src$$XMMRegister);
8387 %}
8388 ins_pipe(pipe_slow); // XXX
8389 %}
8390
8391 // Store immediate double 0.0 (it is faster than store from XMM register)
8392 instruct storeD0_imm(memory mem, immD0 src)
8393 %{
8394 predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
8395 match(Set mem (StoreD mem src));
8396
8397 ins_cost(50);
8398 format %{ "movq $mem, $src\t# double 0." %}
8399 ins_encode %{
8400 __ movq($mem$$Address, $src$$constant);
8401 %}
8402 ins_pipe(ialu_mem_imm);
8403 %}
8404
8405 instruct storeD0(memory mem, immD0 zero)
8406 %{
8407 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8408 match(Set mem (StoreD mem zero));
8409
8410 ins_cost(25); // XXX
8411 format %{ "movq $mem, R12\t# double 0. (R12_heapbase==0)" %}
8412 ins_encode %{
8413 __ movq($mem$$Address, r12);
8414 %}
8415 ins_pipe(ialu_mem_reg);
8416 %}
8417
8418 instruct storeSSI(stackSlotI dst, rRegI src)
8419 %{
8420 match(Set dst src);
8421
8422 ins_cost(100);
8423 format %{ "movl $dst, $src\t# int stk" %}
8424 ins_encode %{
8425 __ movl($dst$$Address, $src$$Register);
8426 %}
8427 ins_pipe( ialu_mem_reg );
8428 %}
8429
8430 instruct storeSSL(stackSlotL dst, rRegL src)
8431 %{
8432 match(Set dst src);
8433
8434 ins_cost(100);
8435 format %{ "movq $dst, $src\t# long stk" %}
8436 ins_encode %{
8437 __ movq($dst$$Address, $src$$Register);
8438 %}
8439 ins_pipe(ialu_mem_reg);
8440 %}
8441
8442 instruct storeSSP(stackSlotP dst, rRegP src)
8443 %{
8444 match(Set dst src);
8445
8446 ins_cost(100);
8447 format %{ "movq $dst, $src\t# ptr stk" %}
8448 ins_encode %{
8449 __ movq($dst$$Address, $src$$Register);
8450 %}
8451 ins_pipe(ialu_mem_reg);
8452 %}
8453
8454 instruct storeSSF(stackSlotF dst, regF src)
8455 %{
8456 match(Set dst src);
8457
8458 ins_cost(95); // XXX
8459 format %{ "movss $dst, $src\t# float stk" %}
8460 ins_encode %{
8461 __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
8462 %}
8463 ins_pipe(pipe_slow); // XXX
8464 %}
8465
8466 instruct storeSSD(stackSlotD dst, regD src)
8467 %{
8468 match(Set dst src);
8469
8470 ins_cost(95); // XXX
8471 format %{ "movsd $dst, $src\t# double stk" %}
8472 ins_encode %{
8473 __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
8474 %}
8475 ins_pipe(pipe_slow); // XXX
8476 %}
8477
8478 instruct cacheWB(indirect addr)
8479 %{
8480 predicate(VM_Version::supports_data_cache_line_flush());
8481 match(CacheWB addr);
8482
8483 ins_cost(100);
8484 format %{"cache wb $addr" %}
8485 ins_encode %{
8486 assert($addr->index_position() < 0, "should be");
8487 assert($addr$$disp == 0, "should be");
8488 __ cache_wb(Address($addr$$base$$Register, 0));
8489 %}
8490 ins_pipe(pipe_slow); // XXX
8491 %}
8492
8493 instruct cacheWBPreSync()
8494 %{
8495 predicate(VM_Version::supports_data_cache_line_flush());
8496 match(CacheWBPreSync);
8497
8498 ins_cost(100);
8499 format %{"cache wb presync" %}
8500 ins_encode %{
8501 __ cache_wbsync(true);
8502 %}
8503 ins_pipe(pipe_slow); // XXX
8504 %}
8505
8506 instruct cacheWBPostSync()
8507 %{
8508 predicate(VM_Version::supports_data_cache_line_flush());
8509 match(CacheWBPostSync);
8510
8511 ins_cost(100);
8512 format %{"cache wb postsync" %}
8513 ins_encode %{
8514 __ cache_wbsync(false);
8515 %}
8516 ins_pipe(pipe_slow); // XXX
8517 %}
8518
8519 //----------BSWAP Instructions-------------------------------------------------
8520 instruct bytes_reverse_int(rRegI dst) %{
8521 match(Set dst (ReverseBytesI dst));
8522
8523 format %{ "bswapl $dst" %}
8524 ins_encode %{
8525 __ bswapl($dst$$Register);
8526 %}
8527 ins_pipe( ialu_reg );
8528 %}
8529
8530 instruct bytes_reverse_long(rRegL dst) %{
8531 match(Set dst (ReverseBytesL dst));
8532
8533 format %{ "bswapq $dst" %}
8534 ins_encode %{
8535 __ bswapq($dst$$Register);
8536 %}
8537 ins_pipe( ialu_reg);
8538 %}
8539
8540 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
8541 match(Set dst (ReverseBytesUS dst));
8542 effect(KILL cr);
8543
8544 format %{ "bswapl $dst\n\t"
8545 "shrl $dst,16\n\t" %}
8546 ins_encode %{
8547 __ bswapl($dst$$Register);
8548 __ shrl($dst$$Register, 16);
8549 %}
8550 ins_pipe( ialu_reg );
8551 %}
8552
8553 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
8554 match(Set dst (ReverseBytesS dst));
8555 effect(KILL cr);
8556
8557 format %{ "bswapl $dst\n\t"
8558 "sar $dst,16\n\t" %}
8559 ins_encode %{
8560 __ bswapl($dst$$Register);
8561 __ sarl($dst$$Register, 16);
8562 %}
8563 ins_pipe( ialu_reg );
8564 %}
8565
8566 //---------- Zeros Count Instructions ------------------------------------------
8567
8568 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
8569 predicate(UseCountLeadingZerosInstruction);
8570 match(Set dst (CountLeadingZerosI src));
8571 effect(KILL cr);
8572
8573 format %{ "lzcntl $dst, $src\t# count leading zeros (int)" %}
8574 ins_encode %{
8575 __ lzcntl($dst$$Register, $src$$Register);
8576 %}
8577 ins_pipe(ialu_reg);
8578 %}
8579
8580 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
8581 predicate(UseCountLeadingZerosInstruction);
8582 match(Set dst (CountLeadingZerosI (LoadI src)));
8583 effect(KILL cr);
8584 ins_cost(175);
8585 format %{ "lzcntl $dst, $src\t# count leading zeros (int)" %}
8586 ins_encode %{
8587 __ lzcntl($dst$$Register, $src$$Address);
8588 %}
8589 ins_pipe(ialu_reg_mem);
8590 %}
8591
8592 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
8593 predicate(!UseCountLeadingZerosInstruction);
8594 match(Set dst (CountLeadingZerosI src));
8595 effect(KILL cr);
8596
8597 format %{ "bsrl $dst, $src\t# count leading zeros (int)\n\t"
8598 "jnz skip\n\t"
8599 "movl $dst, -1\n"
8600 "skip:\n\t"
8601 "negl $dst\n\t"
8602 "addl $dst, 31" %}
8603 ins_encode %{
8604 Register Rdst = $dst$$Register;
8605 Register Rsrc = $src$$Register;
8606 Label skip;
8607 __ bsrl(Rdst, Rsrc);
8608 __ jccb(Assembler::notZero, skip);
8609 __ movl(Rdst, -1);
8610 __ bind(skip);
8611 __ negl(Rdst);
8612 __ addl(Rdst, BitsPerInt - 1);
8613 %}
8614 ins_pipe(ialu_reg);
8615 %}
8616
8617 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
8618 predicate(UseCountLeadingZerosInstruction);
8619 match(Set dst (CountLeadingZerosL src));
8620 effect(KILL cr);
8621
8622 format %{ "lzcntq $dst, $src\t# count leading zeros (long)" %}
8623 ins_encode %{
8624 __ lzcntq($dst$$Register, $src$$Register);
8625 %}
8626 ins_pipe(ialu_reg);
8627 %}
8628
8629 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
8630 predicate(UseCountLeadingZerosInstruction);
8631 match(Set dst (CountLeadingZerosL (LoadL src)));
8632 effect(KILL cr);
8633 ins_cost(175);
8634 format %{ "lzcntq $dst, $src\t# count leading zeros (long)" %}
8635 ins_encode %{
8636 __ lzcntq($dst$$Register, $src$$Address);
8637 %}
8638 ins_pipe(ialu_reg_mem);
8639 %}
8640
8641 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
8642 predicate(!UseCountLeadingZerosInstruction);
8643 match(Set dst (CountLeadingZerosL src));
8644 effect(KILL cr);
8645
8646 format %{ "bsrq $dst, $src\t# count leading zeros (long)\n\t"
8647 "jnz skip\n\t"
8648 "movl $dst, -1\n"
8649 "skip:\n\t"
8650 "negl $dst\n\t"
8651 "addl $dst, 63" %}
8652 ins_encode %{
8653 Register Rdst = $dst$$Register;
8654 Register Rsrc = $src$$Register;
8655 Label skip;
8656 __ bsrq(Rdst, Rsrc);
8657 __ jccb(Assembler::notZero, skip);
8658 __ movl(Rdst, -1);
8659 __ bind(skip);
8660 __ negl(Rdst);
8661 __ addl(Rdst, BitsPerLong - 1);
8662 %}
8663 ins_pipe(ialu_reg);
8664 %}
8665
8666 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
8667 predicate(UseCountTrailingZerosInstruction);
8668 match(Set dst (CountTrailingZerosI src));
8669 effect(KILL cr);
8670
8671 format %{ "tzcntl $dst, $src\t# count trailing zeros (int)" %}
8672 ins_encode %{
8673 __ tzcntl($dst$$Register, $src$$Register);
8674 %}
8675 ins_pipe(ialu_reg);
8676 %}
8677
8678 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
8679 predicate(UseCountTrailingZerosInstruction);
8680 match(Set dst (CountTrailingZerosI (LoadI src)));
8681 effect(KILL cr);
8682 ins_cost(175);
8683 format %{ "tzcntl $dst, $src\t# count trailing zeros (int)" %}
8684 ins_encode %{
8685 __ tzcntl($dst$$Register, $src$$Address);
8686 %}
8687 ins_pipe(ialu_reg_mem);
8688 %}
8689
8690 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
8691 predicate(!UseCountTrailingZerosInstruction);
8692 match(Set dst (CountTrailingZerosI src));
8693 effect(KILL cr);
8694
8695 format %{ "bsfl $dst, $src\t# count trailing zeros (int)\n\t"
8696 "jnz done\n\t"
8697 "movl $dst, 32\n"
8698 "done:" %}
8699 ins_encode %{
8700 Register Rdst = $dst$$Register;
8701 Label done;
8702 __ bsfl(Rdst, $src$$Register);
8703 __ jccb(Assembler::notZero, done);
8704 __ movl(Rdst, BitsPerInt);
8705 __ bind(done);
8706 %}
8707 ins_pipe(ialu_reg);
8708 %}
8709
8710 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
8711 predicate(UseCountTrailingZerosInstruction);
8712 match(Set dst (CountTrailingZerosL src));
8713 effect(KILL cr);
8714
8715 format %{ "tzcntq $dst, $src\t# count trailing zeros (long)" %}
8716 ins_encode %{
8717 __ tzcntq($dst$$Register, $src$$Register);
8718 %}
8719 ins_pipe(ialu_reg);
8720 %}
8721
8722 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
8723 predicate(UseCountTrailingZerosInstruction);
8724 match(Set dst (CountTrailingZerosL (LoadL src)));
8725 effect(KILL cr);
8726 ins_cost(175);
8727 format %{ "tzcntq $dst, $src\t# count trailing zeros (long)" %}
8728 ins_encode %{
8729 __ tzcntq($dst$$Register, $src$$Address);
8730 %}
8731 ins_pipe(ialu_reg_mem);
8732 %}
8733
8734 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
8735 predicate(!UseCountTrailingZerosInstruction);
8736 match(Set dst (CountTrailingZerosL src));
8737 effect(KILL cr);
8738
8739 format %{ "bsfq $dst, $src\t# count trailing zeros (long)\n\t"
8740 "jnz done\n\t"
8741 "movl $dst, 64\n"
8742 "done:" %}
8743 ins_encode %{
8744 Register Rdst = $dst$$Register;
8745 Label done;
8746 __ bsfq(Rdst, $src$$Register);
8747 __ jccb(Assembler::notZero, done);
8748 __ movl(Rdst, BitsPerLong);
8749 __ bind(done);
8750 %}
8751 ins_pipe(ialu_reg);
8752 %}
8753
8754 //--------------- Reverse Operation Instructions ----------------
8755 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
8756 predicate(!VM_Version::supports_gfni());
8757 match(Set dst (ReverseI src));
8758 effect(TEMP dst, TEMP rtmp, KILL cr);
8759 format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
8760 ins_encode %{
8761 __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
8762 %}
8763 ins_pipe( ialu_reg );
8764 %}
8765
8766 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, vlRegF xtmp1, vlRegF xtmp2, rRegL rtmp, rFlagsReg cr) %{
8767 predicate(VM_Version::supports_gfni());
8768 match(Set dst (ReverseI src));
8769 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
8770 format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
8771 ins_encode %{
8772 __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
8773 %}
8774 ins_pipe( ialu_reg );
8775 %}
8776
8777 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
8778 predicate(!VM_Version::supports_gfni());
8779 match(Set dst (ReverseL src));
8780 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
8781 format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
8782 ins_encode %{
8783 __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
8784 %}
8785 ins_pipe( ialu_reg );
8786 %}
8787
8788 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, vlRegD xtmp1, vlRegD xtmp2, rRegL rtmp, rFlagsReg cr) %{
8789 predicate(VM_Version::supports_gfni());
8790 match(Set dst (ReverseL src));
8791 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
8792 format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
8793 ins_encode %{
8794 __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
8795 %}
8796 ins_pipe( ialu_reg );
8797 %}
8798
8799 //---------- Population Count Instructions -------------------------------------
8800
8801 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
8802 predicate(UsePopCountInstruction);
8803 match(Set dst (PopCountI src));
8804 effect(KILL cr);
8805
8806 format %{ "popcnt $dst, $src" %}
8807 ins_encode %{
8808 __ popcntl($dst$$Register, $src$$Register);
8809 %}
8810 ins_pipe(ialu_reg);
8811 %}
8812
8813 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
8814 predicate(UsePopCountInstruction);
8815 match(Set dst (PopCountI (LoadI mem)));
8816 effect(KILL cr);
8817
8818 format %{ "popcnt $dst, $mem" %}
8819 ins_encode %{
8820 __ popcntl($dst$$Register, $mem$$Address);
8821 %}
8822 ins_pipe(ialu_reg);
8823 %}
8824
8825 // Note: Long.bitCount(long) returns an int.
8826 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
8827 predicate(UsePopCountInstruction);
8828 match(Set dst (PopCountL src));
8829 effect(KILL cr);
8830
8831 format %{ "popcnt $dst, $src" %}
8832 ins_encode %{
8833 __ popcntq($dst$$Register, $src$$Register);
8834 %}
8835 ins_pipe(ialu_reg);
8836 %}
8837
8838 // Note: Long.bitCount(long) returns an int.
8839 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
8840 predicate(UsePopCountInstruction);
8841 match(Set dst (PopCountL (LoadL mem)));
8842 effect(KILL cr);
8843
8844 format %{ "popcnt $dst, $mem" %}
8845 ins_encode %{
8846 __ popcntq($dst$$Register, $mem$$Address);
8847 %}
8848 ins_pipe(ialu_reg);
8849 %}
8850
8851
8852 //----------MemBar Instructions-----------------------------------------------
8853 // Memory barrier flavors
8854
8855 instruct membar_acquire()
8856 %{
8857 match(MemBarAcquire);
8858 match(LoadFence);
8859 ins_cost(0);
8860
8861 size(0);
8862 format %{ "MEMBAR-acquire ! (empty encoding)" %}
8863 ins_encode();
8864 ins_pipe(empty);
8865 %}
8866
8867 instruct membar_acquire_lock()
8868 %{
8869 match(MemBarAcquireLock);
8870 ins_cost(0);
8871
8872 size(0);
8873 format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
8874 ins_encode();
8875 ins_pipe(empty);
8876 %}
8877
8878 instruct membar_release()
8879 %{
8880 match(MemBarRelease);
8881 match(StoreFence);
8882 ins_cost(0);
8883
8884 size(0);
8885 format %{ "MEMBAR-release ! (empty encoding)" %}
8886 ins_encode();
8887 ins_pipe(empty);
8888 %}
8889
8890 instruct membar_release_lock()
8891 %{
8892 match(MemBarReleaseLock);
8893 ins_cost(0);
8894
8895 size(0);
8896 format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
8897 ins_encode();
8898 ins_pipe(empty);
8899 %}
8900
8901 instruct membar_storeload(rFlagsReg cr) %{
8902 match(MemBarStoreLoad);
8903 effect(KILL cr);
8904 ins_cost(400);
8905
8906 format %{
8907 $$template
8908 $$emit$$"lock addl [rsp + #0], 0\t! membar_storeload"
8909 %}
8910 ins_encode %{
8911 __ membar(Assembler::StoreLoad);
8912 %}
8913 ins_pipe(pipe_slow);
8914 %}
8915
8916 instruct membar_volatile(rFlagsReg cr) %{
8917 match(MemBarVolatile);
8918 effect(KILL cr);
8919 ins_cost(400);
8920
8921 format %{
8922 $$template
8923 $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
8924 %}
8925 ins_encode %{
8926 __ membar(Assembler::StoreLoad);
8927 %}
8928 ins_pipe(pipe_slow);
8929 %}
8930
8931 instruct unnecessary_membar_volatile()
8932 %{
8933 match(MemBarVolatile);
8934 predicate(Matcher::post_store_load_barrier(n));
8935 ins_cost(0);
8936
8937 size(0);
8938 format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
8939 ins_encode();
8940 ins_pipe(empty);
8941 %}
8942
8943 instruct membar_full(rFlagsReg cr) %{
8944 match(MemBarFull);
8945 effect(KILL cr);
8946 ins_cost(400);
8947
8948 format %{
8949 $$template
8950 $$emit$$"lock addl [rsp + #0], 0\t! membar_full"
8951 %}
8952 ins_encode %{
8953 __ membar(Assembler::StoreLoad);
8954 %}
8955 ins_pipe(pipe_slow);
8956 %}
8957
8958 instruct membar_storestore() %{
8959 match(MemBarStoreStore);
8960 match(StoreStoreFence);
8961 ins_cost(0);
8962
8963 size(0);
8964 format %{ "MEMBAR-storestore (empty encoding)" %}
8965 ins_encode( );
8966 ins_pipe(empty);
8967 %}
8968
8969 //----------Move Instructions--------------------------------------------------
8970
8971 instruct castX2P(rRegP dst, rRegL src)
8972 %{
8973 match(Set dst (CastX2P src));
8974
8975 format %{ "movq $dst, $src\t# long->ptr" %}
8976 ins_encode %{
8977 if ($dst$$reg != $src$$reg) {
8978 __ movptr($dst$$Register, $src$$Register);
8979 }
8980 %}
8981 ins_pipe(ialu_reg_reg); // XXX
8982 %}
8983
8984 instruct castI2N(rRegN dst, rRegI src)
8985 %{
8986 match(Set dst (CastI2N src));
8987
8988 format %{ "movq $dst, $src\t# int -> narrow ptr" %}
8989 ins_encode %{
8990 if ($dst$$reg != $src$$reg) {
8991 __ movl($dst$$Register, $src$$Register);
8992 }
8993 %}
8994 ins_pipe(ialu_reg_reg); // XXX
8995 %}
8996
8997 instruct castN2X(rRegL dst, rRegN src)
8998 %{
8999 match(Set dst (CastP2X src));
9000
9001 format %{ "movq $dst, $src\t# ptr -> long" %}
9002 ins_encode %{
9003 if ($dst$$reg != $src$$reg) {
9004 __ movptr($dst$$Register, $src$$Register);
9005 }
9006 %}
9007 ins_pipe(ialu_reg_reg); // XXX
9008 %}
9009
9010 instruct castP2X(rRegL dst, rRegP src)
9011 %{
9012 match(Set dst (CastP2X src));
9013
9014 format %{ "movq $dst, $src\t# ptr -> long" %}
9015 ins_encode %{
9016 if ($dst$$reg != $src$$reg) {
9017 __ movptr($dst$$Register, $src$$Register);
9018 }
9019 %}
9020 ins_pipe(ialu_reg_reg); // XXX
9021 %}
9022
9023 // Convert oop into int for vectors alignment masking
9024 instruct convP2I(rRegI dst, rRegP src)
9025 %{
9026 match(Set dst (ConvL2I (CastP2X src)));
9027
9028 format %{ "movl $dst, $src\t# ptr -> int" %}
9029 ins_encode %{
9030 __ movl($dst$$Register, $src$$Register);
9031 %}
9032 ins_pipe(ialu_reg_reg); // XXX
9033 %}
9034
9035 // Convert compressed oop into int for vectors alignment masking
9036 // in case of 32bit oops (heap < 4Gb).
9037 instruct convN2I(rRegI dst, rRegN src)
9038 %{
9039 predicate(CompressedOops::shift() == 0);
9040 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
9041
9042 format %{ "movl $dst, $src\t# compressed ptr -> int" %}
9043 ins_encode %{
9044 __ movl($dst$$Register, $src$$Register);
9045 %}
9046 ins_pipe(ialu_reg_reg); // XXX
9047 %}
9048
9049 // Convert oop pointer into compressed form
9050 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
9051 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
9052 match(Set dst (EncodeP src));
9053 effect(KILL cr);
9054 format %{ "encode_heap_oop $dst,$src" %}
9055 ins_encode %{
9056 Register s = $src$$Register;
9057 Register d = $dst$$Register;
9058 if (s != d) {
9059 __ movq(d, s);
9060 }
9061 __ encode_heap_oop(d);
9062 %}
9063 ins_pipe(ialu_reg_long);
9064 %}
9065
9066 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
9067 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
9068 match(Set dst (EncodeP src));
9069 effect(KILL cr);
9070 format %{ "encode_heap_oop_not_null $dst,$src" %}
9071 ins_encode %{
9072 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
9073 %}
9074 ins_pipe(ialu_reg_long);
9075 %}
9076
9077 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
9078 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
9079 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
9080 match(Set dst (DecodeN src));
9081 effect(KILL cr);
9082 format %{ "decode_heap_oop $dst,$src" %}
9083 ins_encode %{
9084 Register s = $src$$Register;
9085 Register d = $dst$$Register;
9086 if (s != d) {
9087 __ movq(d, s);
9088 }
9089 __ decode_heap_oop(d);
9090 %}
9091 ins_pipe(ialu_reg_long);
9092 %}
9093
9094 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
9095 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
9096 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
9097 match(Set dst (DecodeN src));
9098 effect(KILL cr);
9099 format %{ "decode_heap_oop_not_null $dst,$src" %}
9100 ins_encode %{
9101 Register s = $src$$Register;
9102 Register d = $dst$$Register;
9103 if (s != d) {
9104 __ decode_heap_oop_not_null(d, s);
9105 } else {
9106 __ decode_heap_oop_not_null(d);
9107 }
9108 %}
9109 ins_pipe(ialu_reg_long);
9110 %}
9111
9112 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
9113 match(Set dst (EncodePKlass src));
9114 effect(TEMP dst, KILL cr);
9115 format %{ "encode_and_move_klass_not_null $dst,$src" %}
9116 ins_encode %{
9117 __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
9118 %}
9119 ins_pipe(ialu_reg_long);
9120 %}
9121
9122 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
9123 match(Set dst (DecodeNKlass src));
9124 effect(TEMP dst, KILL cr);
9125 format %{ "decode_and_move_klass_not_null $dst,$src" %}
9126 ins_encode %{
9127 __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
9128 %}
9129 ins_pipe(ialu_reg_long);
9130 %}
9131
9132 //----------Conditional Move---------------------------------------------------
9133 // Jump
9134 // dummy instruction for generating temp registers
9135 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
9136 match(Jump (LShiftL switch_val shift));
9137 ins_cost(350);
9138 predicate(false);
9139 effect(TEMP dest);
9140
9141 format %{ "leaq $dest, [$constantaddress]\n\t"
9142 "jmp [$dest + $switch_val << $shift]\n\t" %}
9143 ins_encode %{
9144 // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
9145 // to do that and the compiler is using that register as one it can allocate.
9146 // So we build it all by hand.
9147 // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
9148 // ArrayAddress dispatch(table, index);
9149 Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
9150 __ lea($dest$$Register, $constantaddress);
9151 __ jmp(dispatch);
9152 %}
9153 ins_pipe(pipe_jmp);
9154 %}
9155
9156 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
9157 match(Jump (AddL (LShiftL switch_val shift) offset));
9158 ins_cost(350);
9159 effect(TEMP dest);
9160
9161 format %{ "leaq $dest, [$constantaddress]\n\t"
9162 "jmp [$dest + $switch_val << $shift + $offset]\n\t" %}
9163 ins_encode %{
9164 // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
9165 // to do that and the compiler is using that register as one it can allocate.
9166 // So we build it all by hand.
9167 // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
9168 // ArrayAddress dispatch(table, index);
9169 Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
9170 __ lea($dest$$Register, $constantaddress);
9171 __ jmp(dispatch);
9172 %}
9173 ins_pipe(pipe_jmp);
9174 %}
9175
9176 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
9177 match(Jump switch_val);
9178 ins_cost(350);
9179 effect(TEMP dest);
9180
9181 format %{ "leaq $dest, [$constantaddress]\n\t"
9182 "jmp [$dest + $switch_val]\n\t" %}
9183 ins_encode %{
9184 // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
9185 // to do that and the compiler is using that register as one it can allocate.
9186 // So we build it all by hand.
9187 // Address index(noreg, switch_reg, Address::times_1);
9188 // ArrayAddress dispatch(table, index);
9189 Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
9190 __ lea($dest$$Register, $constantaddress);
9191 __ jmp(dispatch);
9192 %}
9193 ins_pipe(pipe_jmp);
9194 %}
9195
9196 // Conditional move
9197 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
9198 %{
9199 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9200 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9201
9202 ins_cost(100); // XXX
9203 format %{ "setbn$cop $dst\t# signed, int" %}
9204 ins_encode %{
9205 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9206 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9207 %}
9208 ins_pipe(ialu_reg);
9209 %}
9210
9211 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
9212 %{
9213 predicate(!UseAPX);
9214 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9215
9216 ins_cost(200); // XXX
9217 format %{ "cmovl$cop $dst, $src\t# signed, int" %}
9218 ins_encode %{
9219 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9220 %}
9221 ins_pipe(pipe_cmov_reg);
9222 %}
9223
9224 instruct cmovI_reg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr, cmpOp cop)
9225 %{
9226 predicate(UseAPX);
9227 match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
9228
9229 ins_cost(200);
9230 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
9231 ins_encode %{
9232 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9233 %}
9234 ins_pipe(pipe_cmov_reg);
9235 %}
9236
9237 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
9238 %{
9239 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9240 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9241
9242 ins_cost(100); // XXX
9243 format %{ "setbn$cop $dst\t# unsigned, int" %}
9244 ins_encode %{
9245 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9246 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9247 %}
9248 ins_pipe(ialu_reg);
9249 %}
9250
9251 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
9252 predicate(!UseAPX);
9253 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9254
9255 ins_cost(200); // XXX
9256 format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
9257 ins_encode %{
9258 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9259 %}
9260 ins_pipe(pipe_cmov_reg);
9261 %}
9262
9263 instruct cmovI_regU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, rRegI src2) %{
9264 predicate(UseAPX);
9265 match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
9266
9267 ins_cost(200);
9268 format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
9269 ins_encode %{
9270 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9271 %}
9272 ins_pipe(pipe_cmov_reg);
9273 %}
9274
9275 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
9276 %{
9277 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9278 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9279
9280 ins_cost(100); // XXX
9281 format %{ "setbn$cop $dst\t# unsigned, int" %}
9282 ins_encode %{
9283 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9284 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9285 %}
9286 ins_pipe(ialu_reg);
9287 %}
9288
9289 instruct cmovI_imm_01UCFE(rRegI dst, immI_1 src, rFlagsRegUCFE cr, cmpOpUCFE cop)
9290 %{
9291 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9292 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9293
9294 ins_cost(100); // XXX
9295 format %{ "setbn$cop $dst\t# signed, unsigned, int" %}
9296 ins_encode %{
9297 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9298 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9299 %}
9300 ins_pipe(ialu_reg);
9301 %}
9302
9303 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
9304 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9305
9306 ins_cost(200);
9307 expand %{
9308 cmovI_regU(cop, cr, dst, src);
9309 %}
9310 %}
9311
9312 instruct cmovI_regUCFE_ndd(rRegI dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegI src1, rRegI src2) %{
9313 match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
9314
9315 ins_cost(200);
9316 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, unsigned, int ndd" %}
9317 ins_encode %{
9318 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9319 %}
9320 ins_pipe(pipe_cmov_reg);
9321 %}
9322
9323 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
9324 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9325 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9326
9327 ins_cost(200); // XXX
9328 format %{ "cmovpl $dst, $src\n\t"
9329 "cmovnel $dst, $src" %}
9330 ins_encode %{
9331 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9332 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9333 %}
9334 ins_pipe(pipe_cmov_reg);
9335 %}
9336
9337 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9338 // inputs of the CMove
9339 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
9340 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9341 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9342 effect(TEMP dst);
9343
9344 ins_cost(200); // XXX
9345 format %{ "cmovpl $dst, $src\n\t"
9346 "cmovnel $dst, $src" %}
9347 ins_encode %{
9348 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9349 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9350 %}
9351 ins_pipe(pipe_cmov_reg);
9352 %}
9353
9354 // Conditional move
9355 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
9356 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9357
9358 ins_cost(250); // XXX
9359 format %{ "cmovl$cop $dst, $src\t# signed, int" %}
9360 ins_encode %{
9361 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9362 %}
9363 ins_pipe(pipe_cmov_mem);
9364 %}
9365
9366 // Conditional move
9367 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
9368 %{
9369 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9370
9371 ins_cost(250); // XXX
9372 format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
9373 ins_encode %{
9374 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9375 %}
9376 ins_pipe(pipe_cmov_mem);
9377 %}
9378
9379 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
9380 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9381
9382 ins_cost(250);
9383 expand %{
9384 cmovI_memU(cop, cr, dst, src);
9385 %}
9386 %}
9387
9388 instruct cmovI_memUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, rRegI dst, memory src) %{
9389 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9390
9391 ins_cost(250); // XXX
9392 format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
9393 ins_encode %{
9394 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9395 %}
9396 ins_pipe(pipe_cmov_mem);
9397 %}
9398
9399 // Conditional move
9400 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
9401 %{
9402 predicate(!UseAPX);
9403 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9404
9405 ins_cost(200); // XXX
9406 format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
9407 ins_encode %{
9408 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9409 %}
9410 ins_pipe(pipe_cmov_reg);
9411 %}
9412
9413 // Conditional move ndd
9414 instruct cmovN_reg_ndd(rRegN dst, rRegN src1, rRegN src2, rFlagsReg cr, cmpOp cop)
9415 %{
9416 predicate(UseAPX);
9417 match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
9418
9419 ins_cost(200);
9420 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, compressed ptr ndd" %}
9421 ins_encode %{
9422 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9423 %}
9424 ins_pipe(pipe_cmov_reg);
9425 %}
9426
9427 // Conditional move
9428 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
9429 %{
9430 predicate(!UseAPX);
9431 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9432
9433 ins_cost(200); // XXX
9434 format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
9435 ins_encode %{
9436 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9437 %}
9438 ins_pipe(pipe_cmov_reg);
9439 %}
9440
9441 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
9442 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9443
9444 ins_cost(200);
9445 expand %{
9446 cmovN_regU(cop, cr, dst, src);
9447 %}
9448 %}
9449
9450 // Conditional move ndd
9451 instruct cmovN_regU_ndd(rRegN dst, cmpOpU cop, rFlagsRegU cr, rRegN src1, rRegN src2)
9452 %{
9453 predicate(UseAPX);
9454 match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
9455
9456 ins_cost(200);
9457 format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
9458 ins_encode %{
9459 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9460 %}
9461 ins_pipe(pipe_cmov_reg);
9462 %}
9463
9464 instruct cmovN_regUCFE_ndd(rRegN dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegN src1, rRegN src2) %{
9465 match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
9466
9467 ins_cost(200);
9468 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, unsigned, compressed ptr ndd" %}
9469 ins_encode %{
9470 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9471 %}
9472 ins_pipe(pipe_cmov_reg);
9473 %}
9474
9475 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
9476 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9477 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9478
9479 ins_cost(200); // XXX
9480 format %{ "cmovpl $dst, $src\n\t"
9481 "cmovnel $dst, $src" %}
9482 ins_encode %{
9483 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9484 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9485 %}
9486 ins_pipe(pipe_cmov_reg);
9487 %}
9488
9489 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9490 // inputs of the CMove
9491 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
9492 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9493 match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
9494
9495 ins_cost(200); // XXX
9496 format %{ "cmovpl $dst, $src\n\t"
9497 "cmovnel $dst, $src" %}
9498 ins_encode %{
9499 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9500 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9501 %}
9502 ins_pipe(pipe_cmov_reg);
9503 %}
9504
9505 // Conditional move
9506 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
9507 %{
9508 predicate(!UseAPX);
9509 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9510
9511 ins_cost(200); // XXX
9512 format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
9513 ins_encode %{
9514 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9515 %}
9516 ins_pipe(pipe_cmov_reg); // XXX
9517 %}
9518
9519 // Conditional move ndd
9520 instruct cmovP_reg_ndd(rRegP dst, rRegP src1, rRegP src2, rFlagsReg cr, cmpOp cop)
9521 %{
9522 predicate(UseAPX);
9523 match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
9524
9525 ins_cost(200);
9526 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, ptr ndd" %}
9527 ins_encode %{
9528 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9529 %}
9530 ins_pipe(pipe_cmov_reg);
9531 %}
9532
9533 // Conditional move
9534 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
9535 %{
9536 predicate(!UseAPX);
9537 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9538
9539 ins_cost(200); // XXX
9540 format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
9541 ins_encode %{
9542 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9543 %}
9544 ins_pipe(pipe_cmov_reg); // XXX
9545 %}
9546
9547 // Conditional move ndd
9548 instruct cmovP_regU_ndd(rRegP dst, cmpOpU cop, rFlagsRegU cr, rRegP src1, rRegP src2)
9549 %{
9550 predicate(UseAPX);
9551 match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
9552
9553 ins_cost(200);
9554 format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
9555 ins_encode %{
9556 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9557 %}
9558 ins_pipe(pipe_cmov_reg);
9559 %}
9560
9561 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
9562 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9563
9564 ins_cost(200);
9565 expand %{
9566 cmovP_regU(cop, cr, dst, src);
9567 %}
9568 %}
9569
9570 instruct cmovP_regUCFE_ndd(rRegP dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegP src1, rRegP src2) %{
9571 match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
9572
9573 ins_cost(200);
9574 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, unsigned, ptr ndd" %}
9575 ins_encode %{
9576 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9577 %}
9578 ins_pipe(pipe_cmov_reg);
9579 %}
9580
9581 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
9582 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9583 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9584
9585 ins_cost(200); // XXX
9586 format %{ "cmovpq $dst, $src\n\t"
9587 "cmovneq $dst, $src" %}
9588 ins_encode %{
9589 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9590 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9591 %}
9592 ins_pipe(pipe_cmov_reg);
9593 %}
9594
9595 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9596 // inputs of the CMove
9597 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
9598 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9599 match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
9600
9601 ins_cost(200); // XXX
9602 format %{ "cmovpq $dst, $src\n\t"
9603 "cmovneq $dst, $src" %}
9604 ins_encode %{
9605 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9606 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9607 %}
9608 ins_pipe(pipe_cmov_reg);
9609 %}
9610
9611 instruct cmovL_imm_01(rRegL dst, immL1 src, rFlagsReg cr, cmpOp cop)
9612 %{
9613 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9614 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9615
9616 ins_cost(100); // XXX
9617 format %{ "setbn$cop $dst\t# signed, long" %}
9618 ins_encode %{
9619 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9620 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9621 %}
9622 ins_pipe(ialu_reg);
9623 %}
9624
9625 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
9626 %{
9627 predicate(!UseAPX);
9628 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9629
9630 ins_cost(200); // XXX
9631 format %{ "cmovq$cop $dst, $src\t# signed, long" %}
9632 ins_encode %{
9633 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9634 %}
9635 ins_pipe(pipe_cmov_reg); // XXX
9636 %}
9637
9638 instruct cmovL_reg_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, rRegL src2)
9639 %{
9640 predicate(UseAPX);
9641 match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
9642
9643 ins_cost(200);
9644 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
9645 ins_encode %{
9646 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9647 %}
9648 ins_pipe(pipe_cmov_reg);
9649 %}
9650
9651 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
9652 %{
9653 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9654
9655 ins_cost(200); // XXX
9656 format %{ "cmovq$cop $dst, $src\t# signed, long" %}
9657 ins_encode %{
9658 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9659 %}
9660 ins_pipe(pipe_cmov_mem); // XXX
9661 %}
9662
9663 instruct cmovL_imm_01U(rRegL dst, immL1 src, rFlagsRegU cr, cmpOpU cop)
9664 %{
9665 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9666 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9667
9668 ins_cost(100); // XXX
9669 format %{ "setbn$cop $dst\t# unsigned, long" %}
9670 ins_encode %{
9671 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9672 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9673 %}
9674 ins_pipe(ialu_reg);
9675 %}
9676
9677 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
9678 %{
9679 predicate(!UseAPX);
9680 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9681
9682 ins_cost(200); // XXX
9683 format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
9684 ins_encode %{
9685 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9686 %}
9687 ins_pipe(pipe_cmov_reg); // XXX
9688 %}
9689
9690 instruct cmovL_regU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, rRegL src2)
9691 %{
9692 predicate(UseAPX);
9693 match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
9694
9695 ins_cost(200);
9696 format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
9697 ins_encode %{
9698 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9699 %}
9700 ins_pipe(pipe_cmov_reg);
9701 %}
9702
9703 instruct cmovL_imm_01UCF(rRegL dst, immL1 src, rFlagsRegUCF cr, cmpOpUCF cop)
9704 %{
9705 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9706 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9707
9708 ins_cost(100); // XXX
9709 format %{ "setbn$cop $dst\t# unsigned, long" %}
9710 ins_encode %{
9711 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9712 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9713 %}
9714 ins_pipe(ialu_reg);
9715 %}
9716
9717 instruct cmovL_imm_01UCFE(rRegL dst, immL1 src, rFlagsRegUCFE cr, cmpOpUCFE cop)
9718 %{
9719 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9720 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9721
9722 ins_cost(100); // XXX
9723 format %{ "setbn$cop $dst\t# signed, unsigned, long" %}
9724 ins_encode %{
9725 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9726 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9727 %}
9728 ins_pipe(ialu_reg);
9729 %}
9730
9731 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
9732 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9733
9734 ins_cost(200);
9735 expand %{
9736 cmovL_regU(cop, cr, dst, src);
9737 %}
9738 %}
9739
9740 instruct cmovL_regUCFE_ndd(rRegL dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegL src1, rRegL src2)
9741 %{
9742 match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
9743
9744 ins_cost(200);
9745 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, unsigned, long ndd" %}
9746 ins_encode %{
9747 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9748 %}
9749 ins_pipe(pipe_cmov_reg);
9750 %}
9751
9752 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
9753 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9754 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9755
9756 ins_cost(200); // XXX
9757 format %{ "cmovpq $dst, $src\n\t"
9758 "cmovneq $dst, $src" %}
9759 ins_encode %{
9760 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9761 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9762 %}
9763 ins_pipe(pipe_cmov_reg);
9764 %}
9765
9766 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9767 // inputs of the CMove
9768 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
9769 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9770 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9771
9772 ins_cost(200); // XXX
9773 format %{ "cmovpq $dst, $src\n\t"
9774 "cmovneq $dst, $src" %}
9775 ins_encode %{
9776 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9777 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9778 %}
9779 ins_pipe(pipe_cmov_reg);
9780 %}
9781
9782 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
9783 %{
9784 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9785
9786 ins_cost(200); // XXX
9787 format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
9788 ins_encode %{
9789 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9790 %}
9791 ins_pipe(pipe_cmov_mem); // XXX
9792 %}
9793
9794 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
9795 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9796
9797 ins_cost(200);
9798 expand %{
9799 cmovL_memU(cop, cr, dst, src);
9800 %}
9801 %}
9802
9803 instruct cmovL_memUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, rRegL dst, memory src) %{
9804 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9805
9806 ins_cost(200); // XXX
9807 format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
9808 ins_encode %{
9809 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9810 %}
9811 ins_pipe(pipe_cmov_mem); // XXX
9812 %}
9813
9814 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
9815 %{
9816 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9817
9818 ins_cost(200); // XXX
9819 format %{ "jn$cop skip\t# signed cmove float\n\t"
9820 "movss $dst, $src\n"
9821 "skip:" %}
9822 ins_encode %{
9823 Label Lskip;
9824 // Invert sense of branch from sense of CMOV
9825 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9826 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
9827 __ bind(Lskip);
9828 %}
9829 ins_pipe(pipe_slow);
9830 %}
9831
9832 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
9833 %{
9834 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9835
9836 ins_cost(200); // XXX
9837 format %{ "jn$cop skip\t# unsigned cmove float\n\t"
9838 "movss $dst, $src\n"
9839 "skip:" %}
9840 ins_encode %{
9841 Label Lskip;
9842 // Invert sense of branch from sense of CMOV
9843 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9844 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
9845 __ bind(Lskip);
9846 %}
9847 ins_pipe(pipe_slow);
9848 %}
9849
9850 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
9851 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9852
9853 ins_cost(200);
9854 expand %{
9855 cmovF_regU(cop, cr, dst, src);
9856 %}
9857 %}
9858
9859 instruct cmovF_regUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, regF dst, regF src)
9860 %{
9861 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9862
9863 ins_cost(200); // XXX
9864 format %{ "jn$cop skip\t# signed, unsigned cmove float\n\t"
9865 "movss $dst, $src\n"
9866 "skip:" %}
9867 ins_encode %{
9868 Label Lskip;
9869 // Invert sense of branch from sense of CMOV
9870 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9871 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
9872 __ bind(Lskip);
9873 %}
9874 ins_pipe(pipe_slow);
9875 %}
9876
9877 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
9878 %{
9879 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9880
9881 ins_cost(200); // XXX
9882 format %{ "jn$cop skip\t# signed cmove double\n\t"
9883 "movsd $dst, $src\n"
9884 "skip:" %}
9885 ins_encode %{
9886 Label Lskip;
9887 // Invert sense of branch from sense of CMOV
9888 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9889 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
9890 __ bind(Lskip);
9891 %}
9892 ins_pipe(pipe_slow);
9893 %}
9894
9895 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
9896 %{
9897 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9898
9899 ins_cost(200); // XXX
9900 format %{ "jn$cop skip\t# unsigned cmove double\n\t"
9901 "movsd $dst, $src\n"
9902 "skip:" %}
9903 ins_encode %{
9904 Label Lskip;
9905 // Invert sense of branch from sense of CMOV
9906 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9907 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
9908 __ bind(Lskip);
9909 %}
9910 ins_pipe(pipe_slow);
9911 %}
9912
9913 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
9914 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9915
9916 ins_cost(200);
9917 expand %{
9918 cmovD_regU(cop, cr, dst, src);
9919 %}
9920 %}
9921
9922 instruct cmovD_regUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, regD dst, regD src)
9923 %{
9924 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9925
9926 ins_cost(200); // XXX
9927 format %{ "jn$cop skip\t# signed, unsigned cmove double\n\t"
9928 "movsd $dst, $src\n"
9929 "skip:" %}
9930 ins_encode %{
9931 Label Lskip;
9932 // Invert sense of branch from sense of CMOV
9933 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9934 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
9935 __ bind(Lskip);
9936 %}
9937 ins_pipe(pipe_slow);
9938 %}
9939
9940 //----------Arithmetic Instructions--------------------------------------------
9941 //----------Addition Instructions----------------------------------------------
9942
9943 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9944 %{
9945 predicate(!UseAPX);
9946 match(Set dst (AddI dst src));
9947 effect(KILL cr);
9948 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9949 format %{ "addl $dst, $src\t# int" %}
9950 ins_encode %{
9951 __ addl($dst$$Register, $src$$Register);
9952 %}
9953 ins_pipe(ialu_reg_reg);
9954 %}
9955
9956 instruct addI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
9957 %{
9958 predicate(UseAPX);
9959 match(Set dst (AddI src1 src2));
9960 effect(KILL cr);
9961 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
9962
9963 format %{ "eaddl $dst, $src1, $src2\t# int ndd" %}
9964 ins_encode %{
9965 __ eaddl($dst$$Register, $src1$$Register, $src2$$Register, false);
9966 %}
9967 ins_pipe(ialu_reg_reg);
9968 %}
9969
9970 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9971 %{
9972 predicate(!UseAPX);
9973 match(Set dst (AddI dst src));
9974 effect(KILL cr);
9975 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9976
9977 format %{ "addl $dst, $src\t# int" %}
9978 ins_encode %{
9979 __ addl($dst$$Register, $src$$constant);
9980 %}
9981 ins_pipe( ialu_reg );
9982 %}
9983
9984 instruct addI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
9985 %{
9986 predicate(UseAPX);
9987 match(Set dst (AddI src1 src2));
9988 effect(KILL cr);
9989 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
9990
9991 format %{ "eaddl $dst, $src1, $src2\t# int ndd" %}
9992 ins_encode %{
9993 __ eaddl($dst$$Register, $src1$$Register, $src2$$constant, false);
9994 %}
9995 ins_pipe( ialu_reg );
9996 %}
9997
9998 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9999 %{
10000 match(Set dst (AddI dst (LoadI src)));
10001 effect(KILL cr);
10002 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10003
10004 ins_cost(150); // XXX
10005 format %{ "addl $dst, $src\t# int" %}
10006 ins_encode %{
10007 __ addl($dst$$Register, $src$$Address);
10008 %}
10009 ins_pipe(ialu_reg_mem);
10010 %}
10011
10012 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10013 %{
10014 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
10015 effect(KILL cr);
10016 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10017
10018 ins_cost(150); // XXX
10019 format %{ "addl $dst, $src\t# int" %}
10020 ins_encode %{
10021 __ addl($dst$$Address, $src$$Register);
10022 %}
10023 ins_pipe(ialu_mem_reg);
10024 %}
10025
10026 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
10027 %{
10028 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
10029 effect(KILL cr);
10030 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10031
10032
10033 ins_cost(125); // XXX
10034 format %{ "addl $dst, $src\t# int" %}
10035 ins_encode %{
10036 __ addl($dst$$Address, $src$$constant);
10037 %}
10038 ins_pipe(ialu_mem_imm);
10039 %}
10040
10041 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
10042 %{
10043 predicate(!UseAPX && UseIncDec);
10044 match(Set dst (AddI dst src));
10045 effect(KILL cr);
10046
10047 format %{ "incl $dst\t# int" %}
10048 ins_encode %{
10049 __ incrementl($dst$$Register);
10050 %}
10051 ins_pipe(ialu_reg);
10052 %}
10053
10054 instruct incI_rReg_ndd(rRegI dst, rRegI src, immI_1 val, rFlagsReg cr)
10055 %{
10056 predicate(UseAPX && UseIncDec);
10057 match(Set dst (AddI src val));
10058 effect(KILL cr);
10059 flag(PD::Flag_ndd_demotable_opr1);
10060
10061 format %{ "eincl $dst, $src\t# int ndd" %}
10062 ins_encode %{
10063 __ eincl($dst$$Register, $src$$Register, false);
10064 %}
10065 ins_pipe(ialu_reg);
10066 %}
10067
10068 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
10069 %{
10070 predicate(UseIncDec);
10071 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
10072 effect(KILL cr);
10073
10074 ins_cost(125); // XXX
10075 format %{ "incl $dst\t# int" %}
10076 ins_encode %{
10077 __ incrementl($dst$$Address);
10078 %}
10079 ins_pipe(ialu_mem_imm);
10080 %}
10081
10082 // XXX why does that use AddI
10083 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
10084 %{
10085 predicate(!UseAPX && UseIncDec);
10086 match(Set dst (AddI dst src));
10087 effect(KILL cr);
10088
10089 format %{ "decl $dst\t# int" %}
10090 ins_encode %{
10091 __ decrementl($dst$$Register);
10092 %}
10093 ins_pipe(ialu_reg);
10094 %}
10095
10096 instruct decI_rReg_ndd(rRegI dst, rRegI src, immI_M1 val, rFlagsReg cr)
10097 %{
10098 predicate(UseAPX && UseIncDec);
10099 match(Set dst (AddI src val));
10100 effect(KILL cr);
10101 flag(PD::Flag_ndd_demotable_opr1);
10102
10103 format %{ "edecl $dst, $src\t# int ndd" %}
10104 ins_encode %{
10105 __ edecl($dst$$Register, $src$$Register, false);
10106 %}
10107 ins_pipe(ialu_reg);
10108 %}
10109
10110 // XXX why does that use AddI
10111 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
10112 %{
10113 predicate(UseIncDec);
10114 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
10115 effect(KILL cr);
10116
10117 ins_cost(125); // XXX
10118 format %{ "decl $dst\t# int" %}
10119 ins_encode %{
10120 __ decrementl($dst$$Address);
10121 %}
10122 ins_pipe(ialu_mem_imm);
10123 %}
10124
10125 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
10126 %{
10127 predicate(VM_Version::supports_fast_2op_lea());
10128 match(Set dst (AddI (LShiftI index scale) disp));
10129
10130 format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
10131 ins_encode %{
10132 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10133 __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
10134 %}
10135 ins_pipe(ialu_reg_reg);
10136 %}
10137
10138 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
10139 %{
10140 predicate(VM_Version::supports_fast_3op_lea());
10141 match(Set dst (AddI (AddI base index) disp));
10142
10143 format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
10144 ins_encode %{
10145 __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
10146 %}
10147 ins_pipe(ialu_reg_reg);
10148 %}
10149
10150 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
10151 %{
10152 predicate(VM_Version::supports_fast_2op_lea());
10153 match(Set dst (AddI base (LShiftI index scale)));
10154
10155 format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
10156 ins_encode %{
10157 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10158 __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
10159 %}
10160 ins_pipe(ialu_reg_reg);
10161 %}
10162
10163 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
10164 %{
10165 predicate(VM_Version::supports_fast_3op_lea());
10166 match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
10167
10168 format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
10169 ins_encode %{
10170 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10171 __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
10172 %}
10173 ins_pipe(ialu_reg_reg);
10174 %}
10175
10176 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10177 %{
10178 predicate(!UseAPX);
10179 match(Set dst (AddL dst src));
10180 effect(KILL cr);
10181 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10182
10183 format %{ "addq $dst, $src\t# long" %}
10184 ins_encode %{
10185 __ addq($dst$$Register, $src$$Register);
10186 %}
10187 ins_pipe(ialu_reg_reg);
10188 %}
10189
10190 instruct addL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
10191 %{
10192 predicate(UseAPX);
10193 match(Set dst (AddL src1 src2));
10194 effect(KILL cr);
10195 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
10196
10197 format %{ "eaddq $dst, $src1, $src2\t# long ndd" %}
10198 ins_encode %{
10199 __ eaddq($dst$$Register, $src1$$Register, $src2$$Register, false);
10200 %}
10201 ins_pipe(ialu_reg_reg);
10202 %}
10203
10204 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10205 %{
10206 predicate(!UseAPX);
10207 match(Set dst (AddL dst src));
10208 effect(KILL cr);
10209 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10210
10211 format %{ "addq $dst, $src\t# long" %}
10212 ins_encode %{
10213 __ addq($dst$$Register, $src$$constant);
10214 %}
10215 ins_pipe( ialu_reg );
10216 %}
10217
10218 instruct addL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
10219 %{
10220 predicate(UseAPX);
10221 match(Set dst (AddL src1 src2));
10222 effect(KILL cr);
10223 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
10224
10225 format %{ "eaddq $dst, $src1, $src2\t# long ndd" %}
10226 ins_encode %{
10227 __ eaddq($dst$$Register, $src1$$Register, $src2$$constant, false);
10228 %}
10229 ins_pipe( ialu_reg );
10230 %}
10231
10232 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10233 %{
10234 match(Set dst (AddL dst (LoadL src)));
10235 effect(KILL cr);
10236 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10237
10238 ins_cost(150); // XXX
10239 format %{ "addq $dst, $src\t# long" %}
10240 ins_encode %{
10241 __ addq($dst$$Register, $src$$Address);
10242 %}
10243 ins_pipe(ialu_reg_mem);
10244 %}
10245
10246 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10247 %{
10248 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10249 effect(KILL cr);
10250 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10251
10252 ins_cost(150); // XXX
10253 format %{ "addq $dst, $src\t# long" %}
10254 ins_encode %{
10255 __ addq($dst$$Address, $src$$Register);
10256 %}
10257 ins_pipe(ialu_mem_reg);
10258 %}
10259
10260 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10261 %{
10262 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10263 effect(KILL cr);
10264 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10265
10266 ins_cost(125); // XXX
10267 format %{ "addq $dst, $src\t# long" %}
10268 ins_encode %{
10269 __ addq($dst$$Address, $src$$constant);
10270 %}
10271 ins_pipe(ialu_mem_imm);
10272 %}
10273
10274 instruct incL_rReg(rRegL dst, immL1 src, rFlagsReg cr)
10275 %{
10276 predicate(!UseAPX && UseIncDec);
10277 match(Set dst (AddL dst src));
10278 effect(KILL cr);
10279
10280 format %{ "incq $dst\t# long" %}
10281 ins_encode %{
10282 __ incrementq($dst$$Register);
10283 %}
10284 ins_pipe(ialu_reg);
10285 %}
10286
10287 instruct incL_rReg_ndd(rRegL dst, rRegI src, immL1 val, rFlagsReg cr)
10288 %{
10289 predicate(UseAPX && UseIncDec);
10290 match(Set dst (AddL src val));
10291 effect(KILL cr);
10292 flag(PD::Flag_ndd_demotable_opr1);
10293
10294 format %{ "eincq $dst, $src\t# long ndd" %}
10295 ins_encode %{
10296 __ eincq($dst$$Register, $src$$Register, false);
10297 %}
10298 ins_pipe(ialu_reg);
10299 %}
10300
10301 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
10302 %{
10303 predicate(UseIncDec);
10304 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10305 effect(KILL cr);
10306
10307 ins_cost(125); // XXX
10308 format %{ "incq $dst\t# long" %}
10309 ins_encode %{
10310 __ incrementq($dst$$Address);
10311 %}
10312 ins_pipe(ialu_mem_imm);
10313 %}
10314
10315 // XXX why does that use AddL
10316 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
10317 %{
10318 predicate(!UseAPX && UseIncDec);
10319 match(Set dst (AddL dst src));
10320 effect(KILL cr);
10321
10322 format %{ "decq $dst\t# long" %}
10323 ins_encode %{
10324 __ decrementq($dst$$Register);
10325 %}
10326 ins_pipe(ialu_reg);
10327 %}
10328
10329 instruct decL_rReg_ndd(rRegL dst, rRegL src, immL_M1 val, rFlagsReg cr)
10330 %{
10331 predicate(UseAPX && UseIncDec);
10332 match(Set dst (AddL src val));
10333 effect(KILL cr);
10334 flag(PD::Flag_ndd_demotable_opr1);
10335
10336 format %{ "edecq $dst, $src\t# long ndd" %}
10337 ins_encode %{
10338 __ edecq($dst$$Register, $src$$Register, false);
10339 %}
10340 ins_pipe(ialu_reg);
10341 %}
10342
10343 // XXX why does that use AddL
10344 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
10345 %{
10346 predicate(UseIncDec);
10347 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10348 effect(KILL cr);
10349
10350 ins_cost(125); // XXX
10351 format %{ "decq $dst\t# long" %}
10352 ins_encode %{
10353 __ decrementq($dst$$Address);
10354 %}
10355 ins_pipe(ialu_mem_imm);
10356 %}
10357
10358 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
10359 %{
10360 predicate(VM_Version::supports_fast_2op_lea());
10361 match(Set dst (AddL (LShiftL index scale) disp));
10362
10363 format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
10364 ins_encode %{
10365 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10366 __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
10367 %}
10368 ins_pipe(ialu_reg_reg);
10369 %}
10370
10371 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
10372 %{
10373 predicate(VM_Version::supports_fast_3op_lea());
10374 match(Set dst (AddL (AddL base index) disp));
10375
10376 format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
10377 ins_encode %{
10378 __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
10379 %}
10380 ins_pipe(ialu_reg_reg);
10381 %}
10382
10383 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
10384 %{
10385 predicate(VM_Version::supports_fast_2op_lea());
10386 match(Set dst (AddL base (LShiftL index scale)));
10387
10388 format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
10389 ins_encode %{
10390 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10391 __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
10392 %}
10393 ins_pipe(ialu_reg_reg);
10394 %}
10395
10396 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
10397 %{
10398 predicate(VM_Version::supports_fast_3op_lea());
10399 match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
10400
10401 format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
10402 ins_encode %{
10403 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10404 __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
10405 %}
10406 ins_pipe(ialu_reg_reg);
10407 %}
10408
10409 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
10410 %{
10411 match(Set dst (AddP dst src));
10412 effect(KILL cr);
10413 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10414
10415 format %{ "addq $dst, $src\t# ptr" %}
10416 ins_encode %{
10417 __ addq($dst$$Register, $src$$Register);
10418 %}
10419 ins_pipe(ialu_reg_reg);
10420 %}
10421
10422 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
10423 %{
10424 match(Set dst (AddP dst src));
10425 effect(KILL cr);
10426 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10427
10428 format %{ "addq $dst, $src\t# ptr" %}
10429 ins_encode %{
10430 __ addq($dst$$Register, $src$$constant);
10431 %}
10432 ins_pipe( ialu_reg );
10433 %}
10434
10435 // XXX addP mem ops ????
10436
10437 instruct checkCastPP(rRegP dst)
10438 %{
10439 match(Set dst (CheckCastPP dst));
10440
10441 size(0);
10442 format %{ "# checkcastPP of $dst" %}
10443 ins_encode(/* empty encoding */);
10444 ins_pipe(empty);
10445 %}
10446
10447 instruct castPP(rRegP dst)
10448 %{
10449 match(Set dst (CastPP dst));
10450
10451 size(0);
10452 format %{ "# castPP of $dst" %}
10453 ins_encode(/* empty encoding */);
10454 ins_pipe(empty);
10455 %}
10456
10457 instruct castII(rRegI dst)
10458 %{
10459 predicate(VerifyConstraintCasts == 0);
10460 match(Set dst (CastII dst));
10461
10462 size(0);
10463 format %{ "# castII of $dst" %}
10464 ins_encode(/* empty encoding */);
10465 ins_cost(0);
10466 ins_pipe(empty);
10467 %}
10468
10469 instruct castII_checked(rRegI dst, rFlagsReg cr)
10470 %{
10471 predicate(VerifyConstraintCasts > 0);
10472 match(Set dst (CastII dst));
10473
10474 effect(KILL cr);
10475 format %{ "# cast_checked_II $dst" %}
10476 ins_encode %{
10477 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register);
10478 %}
10479 ins_pipe(pipe_slow);
10480 %}
10481
10482 instruct castLL(rRegL dst)
10483 %{
10484 predicate(VerifyConstraintCasts == 0);
10485 match(Set dst (CastLL dst));
10486
10487 size(0);
10488 format %{ "# castLL of $dst" %}
10489 ins_encode(/* empty encoding */);
10490 ins_cost(0);
10491 ins_pipe(empty);
10492 %}
10493
10494 instruct castLL_checked_L32(rRegL dst, rFlagsReg cr)
10495 %{
10496 predicate(VerifyConstraintCasts > 0 && castLL_is_imm32(n));
10497 match(Set dst (CastLL dst));
10498
10499 effect(KILL cr);
10500 format %{ "# cast_checked_LL $dst" %}
10501 ins_encode %{
10502 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, noreg);
10503 %}
10504 ins_pipe(pipe_slow);
10505 %}
10506
10507 instruct castLL_checked(rRegL dst, rRegL tmp, rFlagsReg cr)
10508 %{
10509 predicate(VerifyConstraintCasts > 0 && !castLL_is_imm32(n));
10510 match(Set dst (CastLL dst));
10511
10512 effect(KILL cr, TEMP tmp);
10513 format %{ "# cast_checked_LL $dst\tusing $tmp as TEMP" %}
10514 ins_encode %{
10515 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, $tmp$$Register);
10516 %}
10517 ins_pipe(pipe_slow);
10518 %}
10519
10520 instruct castFF(regF dst)
10521 %{
10522 match(Set dst (CastFF dst));
10523
10524 size(0);
10525 format %{ "# castFF of $dst" %}
10526 ins_encode(/* empty encoding */);
10527 ins_cost(0);
10528 ins_pipe(empty);
10529 %}
10530
10531 instruct castHH(regF dst)
10532 %{
10533 match(Set dst (CastHH dst));
10534
10535 size(0);
10536 format %{ "# castHH of $dst" %}
10537 ins_encode(/* empty encoding */);
10538 ins_cost(0);
10539 ins_pipe(empty);
10540 %}
10541
10542 instruct castDD(regD dst)
10543 %{
10544 match(Set dst (CastDD dst));
10545
10546 size(0);
10547 format %{ "# castDD of $dst" %}
10548 ins_encode(/* empty encoding */);
10549 ins_cost(0);
10550 ins_pipe(empty);
10551 %}
10552
10553 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
10554 instruct compareAndSwapP(rRegI res,
10555 memory mem_ptr,
10556 rax_RegP oldval, rRegP newval,
10557 rFlagsReg cr)
10558 %{
10559 predicate(n->as_LoadStore()->barrier_data() == 0);
10560 match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
10561 match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
10562 effect(KILL cr, KILL oldval);
10563
10564 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10565 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10566 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10567 ins_encode %{
10568 __ lock();
10569 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10570 __ setcc(Assembler::equal, $res$$Register);
10571 %}
10572 ins_pipe( pipe_cmpxchg );
10573 %}
10574
10575 instruct compareAndSwapL(rRegI res,
10576 memory mem_ptr,
10577 rax_RegL oldval, rRegL newval,
10578 rFlagsReg cr)
10579 %{
10580 match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
10581 match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
10582 effect(KILL cr, KILL oldval);
10583
10584 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10585 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10586 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10587 ins_encode %{
10588 __ lock();
10589 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10590 __ setcc(Assembler::equal, $res$$Register);
10591 %}
10592 ins_pipe( pipe_cmpxchg );
10593 %}
10594
10595 instruct compareAndSwapI(rRegI res,
10596 memory mem_ptr,
10597 rax_RegI oldval, rRegI newval,
10598 rFlagsReg cr)
10599 %{
10600 match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
10601 match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
10602 effect(KILL cr, KILL oldval);
10603
10604 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10605 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10606 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10607 ins_encode %{
10608 __ lock();
10609 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10610 __ setcc(Assembler::equal, $res$$Register);
10611 %}
10612 ins_pipe( pipe_cmpxchg );
10613 %}
10614
10615 instruct compareAndSwapB(rRegI res,
10616 memory mem_ptr,
10617 rax_RegI oldval, rRegI newval,
10618 rFlagsReg cr)
10619 %{
10620 match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
10621 match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
10622 effect(KILL cr, KILL oldval);
10623
10624 format %{ "cmpxchgb $mem_ptr,$newval\t# "
10625 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10626 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10627 ins_encode %{
10628 __ lock();
10629 __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
10630 __ setcc(Assembler::equal, $res$$Register);
10631 %}
10632 ins_pipe( pipe_cmpxchg );
10633 %}
10634
10635 instruct compareAndSwapS(rRegI res,
10636 memory mem_ptr,
10637 rax_RegI oldval, rRegI newval,
10638 rFlagsReg cr)
10639 %{
10640 match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
10641 match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
10642 effect(KILL cr, KILL oldval);
10643
10644 format %{ "cmpxchgw $mem_ptr,$newval\t# "
10645 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10646 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10647 ins_encode %{
10648 __ lock();
10649 __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
10650 __ setcc(Assembler::equal, $res$$Register);
10651 %}
10652 ins_pipe( pipe_cmpxchg );
10653 %}
10654
10655 instruct compareAndSwapN(rRegI res,
10656 memory mem_ptr,
10657 rax_RegN oldval, rRegN newval,
10658 rFlagsReg cr) %{
10659 predicate(n->as_LoadStore()->barrier_data() == 0);
10660 match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
10661 match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
10662 effect(KILL cr, KILL oldval);
10663
10664 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10665 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10666 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10667 ins_encode %{
10668 __ lock();
10669 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10670 __ setcc(Assembler::equal, $res$$Register);
10671 %}
10672 ins_pipe( pipe_cmpxchg );
10673 %}
10674
10675 instruct compareAndExchangeB(
10676 memory mem_ptr,
10677 rax_RegI oldval, rRegI newval,
10678 rFlagsReg cr)
10679 %{
10680 match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
10681 effect(KILL cr);
10682
10683 format %{ "cmpxchgb $mem_ptr,$newval\t# "
10684 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10685 ins_encode %{
10686 __ lock();
10687 __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
10688 %}
10689 ins_pipe( pipe_cmpxchg );
10690 %}
10691
10692 instruct compareAndExchangeS(
10693 memory mem_ptr,
10694 rax_RegI oldval, rRegI newval,
10695 rFlagsReg cr)
10696 %{
10697 match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
10698 effect(KILL cr);
10699
10700 format %{ "cmpxchgw $mem_ptr,$newval\t# "
10701 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10702 ins_encode %{
10703 __ lock();
10704 __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
10705 %}
10706 ins_pipe( pipe_cmpxchg );
10707 %}
10708
10709 instruct compareAndExchangeI(
10710 memory mem_ptr,
10711 rax_RegI oldval, rRegI newval,
10712 rFlagsReg cr)
10713 %{
10714 match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
10715 effect(KILL cr);
10716
10717 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10718 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10719 ins_encode %{
10720 __ lock();
10721 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10722 %}
10723 ins_pipe( pipe_cmpxchg );
10724 %}
10725
10726 instruct compareAndExchangeL(
10727 memory mem_ptr,
10728 rax_RegL oldval, rRegL newval,
10729 rFlagsReg cr)
10730 %{
10731 match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
10732 effect(KILL cr);
10733
10734 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10735 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10736 ins_encode %{
10737 __ lock();
10738 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10739 %}
10740 ins_pipe( pipe_cmpxchg );
10741 %}
10742
10743 instruct compareAndExchangeN(
10744 memory mem_ptr,
10745 rax_RegN oldval, rRegN newval,
10746 rFlagsReg cr) %{
10747 predicate(n->as_LoadStore()->barrier_data() == 0);
10748 match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
10749 effect(KILL cr);
10750
10751 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10752 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10753 ins_encode %{
10754 __ lock();
10755 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10756 %}
10757 ins_pipe( pipe_cmpxchg );
10758 %}
10759
10760 instruct compareAndExchangeP(
10761 memory mem_ptr,
10762 rax_RegP oldval, rRegP newval,
10763 rFlagsReg cr)
10764 %{
10765 predicate(n->as_LoadStore()->barrier_data() == 0);
10766 match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
10767 effect(KILL cr);
10768
10769 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10770 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10771 ins_encode %{
10772 __ lock();
10773 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10774 %}
10775 ins_pipe( pipe_cmpxchg );
10776 %}
10777
10778 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
10779 predicate(n->as_LoadStore()->result_not_used());
10780 match(Set dummy (GetAndAddB mem add));
10781 effect(KILL cr);
10782 format %{ "addb_lock $mem, $add" %}
10783 ins_encode %{
10784 __ lock();
10785 __ addb($mem$$Address, $add$$Register);
10786 %}
10787 ins_pipe(pipe_cmpxchg);
10788 %}
10789
10790 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
10791 predicate(n->as_LoadStore()->result_not_used());
10792 match(Set dummy (GetAndAddB mem add));
10793 effect(KILL cr);
10794 format %{ "addb_lock $mem, $add" %}
10795 ins_encode %{
10796 __ lock();
10797 __ addb($mem$$Address, $add$$constant);
10798 %}
10799 ins_pipe(pipe_cmpxchg);
10800 %}
10801
10802 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
10803 predicate(!n->as_LoadStore()->result_not_used());
10804 match(Set newval (GetAndAddB mem newval));
10805 effect(KILL cr);
10806 format %{ "xaddb_lock $mem, $newval\t# $newval -> byte" %}
10807 ins_encode %{
10808 __ lock();
10809 __ xaddb($mem$$Address, $newval$$Register);
10810 __ narrow_subword_type($newval$$Register, T_BYTE);
10811 %}
10812 ins_pipe(pipe_cmpxchg);
10813 %}
10814
10815 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
10816 predicate(n->as_LoadStore()->result_not_used());
10817 match(Set dummy (GetAndAddS mem add));
10818 effect(KILL cr);
10819 format %{ "addw_lock $mem, $add" %}
10820 ins_encode %{
10821 __ lock();
10822 __ addw($mem$$Address, $add$$Register);
10823 %}
10824 ins_pipe(pipe_cmpxchg);
10825 %}
10826
10827 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
10828 predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
10829 match(Set dummy (GetAndAddS mem add));
10830 effect(KILL cr);
10831 format %{ "addw_lock $mem, $add" %}
10832 ins_encode %{
10833 __ lock();
10834 __ addw($mem$$Address, $add$$constant);
10835 %}
10836 ins_pipe(pipe_cmpxchg);
10837 %}
10838
10839 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
10840 predicate(!n->as_LoadStore()->result_not_used());
10841 match(Set newval (GetAndAddS mem newval));
10842 effect(KILL cr);
10843 format %{ "xaddw_lock $mem, $newval\t# $newval -> short" %}
10844 ins_encode %{
10845 __ lock();
10846 __ xaddw($mem$$Address, $newval$$Register);
10847 __ narrow_subword_type($newval$$Register, T_SHORT);
10848 %}
10849 ins_pipe(pipe_cmpxchg);
10850 %}
10851
10852 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
10853 predicate(n->as_LoadStore()->result_not_used());
10854 match(Set dummy (GetAndAddI mem add));
10855 effect(KILL cr);
10856 format %{ "addl_lock $mem, $add" %}
10857 ins_encode %{
10858 __ lock();
10859 __ addl($mem$$Address, $add$$Register);
10860 %}
10861 ins_pipe(pipe_cmpxchg);
10862 %}
10863
10864 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
10865 predicate(n->as_LoadStore()->result_not_used());
10866 match(Set dummy (GetAndAddI mem add));
10867 effect(KILL cr);
10868 format %{ "addl_lock $mem, $add" %}
10869 ins_encode %{
10870 __ lock();
10871 __ addl($mem$$Address, $add$$constant);
10872 %}
10873 ins_pipe(pipe_cmpxchg);
10874 %}
10875
10876 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
10877 predicate(!n->as_LoadStore()->result_not_used());
10878 match(Set newval (GetAndAddI mem newval));
10879 effect(KILL cr);
10880 format %{ "xaddl_lock $mem, $newval" %}
10881 ins_encode %{
10882 __ lock();
10883 __ xaddl($mem$$Address, $newval$$Register);
10884 %}
10885 ins_pipe(pipe_cmpxchg);
10886 %}
10887
10888 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
10889 predicate(n->as_LoadStore()->result_not_used());
10890 match(Set dummy (GetAndAddL mem add));
10891 effect(KILL cr);
10892 format %{ "addq_lock $mem, $add" %}
10893 ins_encode %{
10894 __ lock();
10895 __ addq($mem$$Address, $add$$Register);
10896 %}
10897 ins_pipe(pipe_cmpxchg);
10898 %}
10899
10900 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
10901 predicate(n->as_LoadStore()->result_not_used());
10902 match(Set dummy (GetAndAddL mem add));
10903 effect(KILL cr);
10904 format %{ "addq_lock $mem, $add" %}
10905 ins_encode %{
10906 __ lock();
10907 __ addq($mem$$Address, $add$$constant);
10908 %}
10909 ins_pipe(pipe_cmpxchg);
10910 %}
10911
10912 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
10913 predicate(!n->as_LoadStore()->result_not_used());
10914 match(Set newval (GetAndAddL mem newval));
10915 effect(KILL cr);
10916 format %{ "xaddq_lock $mem, $newval" %}
10917 ins_encode %{
10918 __ lock();
10919 __ xaddq($mem$$Address, $newval$$Register);
10920 %}
10921 ins_pipe(pipe_cmpxchg);
10922 %}
10923
10924 instruct xchgB( memory mem, rRegI newval) %{
10925 match(Set newval (GetAndSetB mem newval));
10926 format %{ "XCHGB $newval,[$mem]\t# $newval -> byte" %}
10927 ins_encode %{
10928 __ xchgb($newval$$Register, $mem$$Address);
10929 __ narrow_subword_type($newval$$Register, T_BYTE);
10930 %}
10931 ins_pipe( pipe_cmpxchg );
10932 %}
10933
10934 instruct xchgS( memory mem, rRegI newval) %{
10935 match(Set newval (GetAndSetS mem newval));
10936 format %{ "XCHGW $newval,[$mem]\t# $newval -> short" %}
10937 ins_encode %{
10938 __ xchgw($newval$$Register, $mem$$Address);
10939 __ narrow_subword_type($newval$$Register, T_SHORT);
10940 %}
10941 ins_pipe( pipe_cmpxchg );
10942 %}
10943
10944 instruct xchgI( memory mem, rRegI newval) %{
10945 match(Set newval (GetAndSetI mem newval));
10946 format %{ "XCHGL $newval,[$mem]" %}
10947 ins_encode %{
10948 __ xchgl($newval$$Register, $mem$$Address);
10949 %}
10950 ins_pipe( pipe_cmpxchg );
10951 %}
10952
10953 instruct xchgL( memory mem, rRegL newval) %{
10954 match(Set newval (GetAndSetL mem newval));
10955 format %{ "XCHGL $newval,[$mem]" %}
10956 ins_encode %{
10957 __ xchgq($newval$$Register, $mem$$Address);
10958 %}
10959 ins_pipe( pipe_cmpxchg );
10960 %}
10961
10962 instruct xchgP( memory mem, rRegP newval) %{
10963 match(Set newval (GetAndSetP mem newval));
10964 predicate(n->as_LoadStore()->barrier_data() == 0);
10965 format %{ "XCHGQ $newval,[$mem]" %}
10966 ins_encode %{
10967 __ xchgq($newval$$Register, $mem$$Address);
10968 %}
10969 ins_pipe( pipe_cmpxchg );
10970 %}
10971
10972 instruct xchgN( memory mem, rRegN newval) %{
10973 predicate(n->as_LoadStore()->barrier_data() == 0);
10974 match(Set newval (GetAndSetN mem newval));
10975 format %{ "XCHGL $newval,$mem]" %}
10976 ins_encode %{
10977 __ xchgl($newval$$Register, $mem$$Address);
10978 %}
10979 ins_pipe( pipe_cmpxchg );
10980 %}
10981
10982 //----------Abs Instructions-------------------------------------------
10983
10984 // Integer Absolute Instructions
10985 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10986 %{
10987 match(Set dst (AbsI src));
10988 effect(TEMP dst, KILL cr);
10989 format %{ "xorl $dst, $dst\t# abs int\n\t"
10990 "subl $dst, $src\n\t"
10991 "cmovll $dst, $src" %}
10992 ins_encode %{
10993 __ xorl($dst$$Register, $dst$$Register);
10994 __ subl($dst$$Register, $src$$Register);
10995 __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
10996 %}
10997
10998 ins_pipe(ialu_reg_reg);
10999 %}
11000
11001 // Long Absolute Instructions
11002 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11003 %{
11004 match(Set dst (AbsL src));
11005 effect(TEMP dst, KILL cr);
11006 format %{ "xorl $dst, $dst\t# abs long\n\t"
11007 "subq $dst, $src\n\t"
11008 "cmovlq $dst, $src" %}
11009 ins_encode %{
11010 __ xorl($dst$$Register, $dst$$Register);
11011 __ subq($dst$$Register, $src$$Register);
11012 __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
11013 %}
11014
11015 ins_pipe(ialu_reg_reg);
11016 %}
11017
11018 //----------Subtraction Instructions-------------------------------------------
11019
11020 // Integer Subtraction Instructions
11021 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
11022 %{
11023 predicate(!UseAPX);
11024 match(Set dst (SubI dst src));
11025 effect(KILL cr);
11026 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11027
11028 format %{ "subl $dst, $src\t# int" %}
11029 ins_encode %{
11030 __ subl($dst$$Register, $src$$Register);
11031 %}
11032 ins_pipe(ialu_reg_reg);
11033 %}
11034
11035 instruct subI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
11036 %{
11037 predicate(UseAPX);
11038 match(Set dst (SubI src1 src2));
11039 effect(KILL cr);
11040 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11041
11042 format %{ "esubl $dst, $src1, $src2\t# int ndd" %}
11043 ins_encode %{
11044 __ esubl($dst$$Register, $src1$$Register, $src2$$Register, false);
11045 %}
11046 ins_pipe(ialu_reg_reg);
11047 %}
11048
11049 instruct subI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
11050 %{
11051 predicate(UseAPX);
11052 match(Set dst (SubI src1 src2));
11053 effect(KILL cr);
11054 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11055
11056 format %{ "esubl $dst, $src1, $src2\t# int ndd" %}
11057 ins_encode %{
11058 __ esubl($dst$$Register, $src1$$Register, $src2$$constant, false);
11059 %}
11060 ins_pipe(ialu_reg_reg);
11061 %}
11062
11063 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
11064 %{
11065 match(Set dst (SubI dst (LoadI src)));
11066 effect(KILL cr);
11067 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11068
11069 ins_cost(150);
11070 format %{ "subl $dst, $src\t# int" %}
11071 ins_encode %{
11072 __ subl($dst$$Register, $src$$Address);
11073 %}
11074 ins_pipe(ialu_reg_mem);
11075 %}
11076
11077 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
11078 %{
11079 match(Set dst (StoreI dst (SubI (LoadI dst) src)));
11080 effect(KILL cr);
11081 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11082
11083 ins_cost(150);
11084 format %{ "subl $dst, $src\t# int" %}
11085 ins_encode %{
11086 __ subl($dst$$Address, $src$$Register);
11087 %}
11088 ins_pipe(ialu_mem_reg);
11089 %}
11090
11091 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11092 %{
11093 predicate(!UseAPX);
11094 match(Set dst (SubL dst src));
11095 effect(KILL cr);
11096 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11097
11098 format %{ "subq $dst, $src\t# long" %}
11099 ins_encode %{
11100 __ subq($dst$$Register, $src$$Register);
11101 %}
11102 ins_pipe(ialu_reg_reg);
11103 %}
11104
11105 instruct subL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11106 %{
11107 predicate(UseAPX);
11108 match(Set dst (SubL src1 src2));
11109 effect(KILL cr);
11110 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11111
11112 format %{ "esubq $dst, $src1, $src2\t# long ndd" %}
11113 ins_encode %{
11114 __ esubq($dst$$Register, $src1$$Register, $src2$$Register, false);
11115 %}
11116 ins_pipe(ialu_reg_reg);
11117 %}
11118
11119 instruct subL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11120 %{
11121 predicate(UseAPX);
11122 match(Set dst (SubL src1 src2));
11123 effect(KILL cr);
11124 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11125
11126 format %{ "esubq $dst, $src1, $src2\t# long ndd" %}
11127 ins_encode %{
11128 __ esubq($dst$$Register, $src1$$Register, $src2$$constant, false);
11129 %}
11130 ins_pipe(ialu_reg_reg);
11131 %}
11132
11133 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11134 %{
11135 match(Set dst (SubL dst (LoadL src)));
11136 effect(KILL cr);
11137 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11138
11139 ins_cost(150);
11140 format %{ "subq $dst, $src\t# long" %}
11141 ins_encode %{
11142 __ subq($dst$$Register, $src$$Address);
11143 %}
11144 ins_pipe(ialu_reg_mem);
11145 %}
11146
11147 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11148 %{
11149 match(Set dst (StoreL dst (SubL (LoadL dst) src)));
11150 effect(KILL cr);
11151 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11152
11153 ins_cost(150);
11154 format %{ "subq $dst, $src\t# long" %}
11155 ins_encode %{
11156 __ subq($dst$$Address, $src$$Register);
11157 %}
11158 ins_pipe(ialu_mem_reg);
11159 %}
11160
11161 // Subtract from a pointer
11162 // XXX hmpf???
11163 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
11164 %{
11165 match(Set dst (AddP dst (SubI zero src)));
11166 effect(KILL cr);
11167
11168 format %{ "subq $dst, $src\t# ptr - int" %}
11169 ins_encode %{
11170 __ subq($dst$$Register, $src$$Register);
11171 %}
11172 ins_pipe(ialu_reg_reg);
11173 %}
11174
11175 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
11176 %{
11177 predicate(!UseAPX);
11178 match(Set dst (SubI zero dst));
11179 effect(KILL cr);
11180 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11181
11182 format %{ "negl $dst\t# int" %}
11183 ins_encode %{
11184 __ negl($dst$$Register);
11185 %}
11186 ins_pipe(ialu_reg);
11187 %}
11188
11189 instruct negI_rReg_ndd(rRegI dst, rRegI src, immI_0 zero, rFlagsReg cr)
11190 %{
11191 predicate(UseAPX);
11192 match(Set dst (SubI zero src));
11193 effect(KILL cr);
11194 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr2);
11195
11196 format %{ "enegl $dst, $src\t# int ndd" %}
11197 ins_encode %{
11198 __ enegl($dst$$Register, $src$$Register, false);
11199 %}
11200 ins_pipe(ialu_reg);
11201 %}
11202
11203 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
11204 %{
11205 predicate(!UseAPX);
11206 match(Set dst (NegI dst));
11207 effect(KILL cr);
11208 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11209
11210 format %{ "negl $dst\t# int" %}
11211 ins_encode %{
11212 __ negl($dst$$Register);
11213 %}
11214 ins_pipe(ialu_reg);
11215 %}
11216
11217 instruct negI_rReg_2_ndd(rRegI dst, rRegI src, rFlagsReg cr)
11218 %{
11219 predicate(UseAPX);
11220 match(Set dst (NegI src));
11221 effect(KILL cr);
11222 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11223
11224 format %{ "enegl $dst, $src\t# int ndd" %}
11225 ins_encode %{
11226 __ enegl($dst$$Register, $src$$Register, false);
11227 %}
11228 ins_pipe(ialu_reg);
11229 %}
11230
11231 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
11232 %{
11233 match(Set dst (StoreI dst (SubI zero (LoadI dst))));
11234 effect(KILL cr);
11235 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11236
11237 format %{ "negl $dst\t# int" %}
11238 ins_encode %{
11239 __ negl($dst$$Address);
11240 %}
11241 ins_pipe(ialu_reg);
11242 %}
11243
11244 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
11245 %{
11246 predicate(!UseAPX);
11247 match(Set dst (SubL zero dst));
11248 effect(KILL cr);
11249 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11250
11251 format %{ "negq $dst\t# long" %}
11252 ins_encode %{
11253 __ negq($dst$$Register);
11254 %}
11255 ins_pipe(ialu_reg);
11256 %}
11257
11258 instruct negL_rReg_ndd(rRegL dst, rRegL src, immL0 zero, rFlagsReg cr)
11259 %{
11260 predicate(UseAPX);
11261 match(Set dst (SubL zero src));
11262 effect(KILL cr);
11263 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr2);
11264
11265 format %{ "enegq $dst, $src\t# long ndd" %}
11266 ins_encode %{
11267 __ enegq($dst$$Register, $src$$Register, false);
11268 %}
11269 ins_pipe(ialu_reg);
11270 %}
11271
11272 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
11273 %{
11274 predicate(!UseAPX);
11275 match(Set dst (NegL dst));
11276 effect(KILL cr);
11277 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11278
11279 format %{ "negq $dst\t# int" %}
11280 ins_encode %{
11281 __ negq($dst$$Register);
11282 %}
11283 ins_pipe(ialu_reg);
11284 %}
11285
11286 instruct negL_rReg_2_ndd(rRegL dst, rRegL src, rFlagsReg cr)
11287 %{
11288 predicate(UseAPX);
11289 match(Set dst (NegL src));
11290 effect(KILL cr);
11291 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11292
11293 format %{ "enegq $dst, $src\t# long ndd" %}
11294 ins_encode %{
11295 __ enegq($dst$$Register, $src$$Register, false);
11296 %}
11297 ins_pipe(ialu_reg);
11298 %}
11299
11300 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
11301 %{
11302 match(Set dst (StoreL dst (SubL zero (LoadL dst))));
11303 effect(KILL cr);
11304 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11305
11306 format %{ "negq $dst\t# long" %}
11307 ins_encode %{
11308 __ negq($dst$$Address);
11309 %}
11310 ins_pipe(ialu_reg);
11311 %}
11312
11313 //----------Multiplication/Division Instructions-------------------------------
11314 // Integer Multiplication Instructions
11315 // Multiply Register
11316
11317 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
11318 %{
11319 predicate(!UseAPX);
11320 match(Set dst (MulI dst src));
11321 effect(KILL cr);
11322
11323 ins_cost(300);
11324 format %{ "imull $dst, $src\t# int" %}
11325 ins_encode %{
11326 __ imull($dst$$Register, $src$$Register);
11327 %}
11328 ins_pipe(ialu_reg_reg_alu0);
11329 %}
11330
11331 instruct mulI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
11332 %{
11333 predicate(UseAPX);
11334 match(Set dst (MulI src1 src2));
11335 effect(KILL cr);
11336 flag(PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
11337
11338 ins_cost(300);
11339 format %{ "eimull $dst, $src1, $src2\t# int ndd" %}
11340 ins_encode %{
11341 __ eimull($dst$$Register, $src1$$Register, $src2$$Register, false);
11342 %}
11343 ins_pipe(ialu_reg_reg_alu0);
11344 %}
11345
11346 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
11347 %{
11348 match(Set dst (MulI src imm));
11349 effect(KILL cr);
11350
11351 ins_cost(300);
11352 format %{ "imull $dst, $src, $imm\t# int" %}
11353 ins_encode %{
11354 __ imull($dst$$Register, $src$$Register, $imm$$constant);
11355 %}
11356 ins_pipe(ialu_reg_reg_alu0);
11357 %}
11358
11359 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
11360 %{
11361 match(Set dst (MulI dst (LoadI src)));
11362 effect(KILL cr);
11363
11364 ins_cost(350);
11365 format %{ "imull $dst, $src\t# int" %}
11366 ins_encode %{
11367 __ imull($dst$$Register, $src$$Address);
11368 %}
11369 ins_pipe(ialu_reg_mem_alu0);
11370 %}
11371
11372 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
11373 %{
11374 match(Set dst (MulI (LoadI src) imm));
11375 effect(KILL cr);
11376
11377 ins_cost(300);
11378 format %{ "imull $dst, $src, $imm\t# int" %}
11379 ins_encode %{
11380 __ imull($dst$$Register, $src$$Address, $imm$$constant);
11381 %}
11382 ins_pipe(ialu_reg_mem_alu0);
11383 %}
11384
11385 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
11386 %{
11387 match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
11388 effect(KILL cr, KILL src2);
11389
11390 expand %{ mulI_rReg(dst, src1, cr);
11391 mulI_rReg(src2, src3, cr);
11392 addI_rReg(dst, src2, cr); %}
11393 %}
11394
11395 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11396 %{
11397 predicate(!UseAPX);
11398 match(Set dst (MulL dst src));
11399 effect(KILL cr);
11400
11401 ins_cost(300);
11402 format %{ "imulq $dst, $src\t# long" %}
11403 ins_encode %{
11404 __ imulq($dst$$Register, $src$$Register);
11405 %}
11406 ins_pipe(ialu_reg_reg_alu0);
11407 %}
11408
11409 instruct mulL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11410 %{
11411 predicate(UseAPX);
11412 match(Set dst (MulL src1 src2));
11413 effect(KILL cr);
11414 flag(PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
11415
11416 ins_cost(300);
11417 format %{ "eimulq $dst, $src1, $src2\t# long ndd" %}
11418 ins_encode %{
11419 __ eimulq($dst$$Register, $src1$$Register, $src2$$Register, false);
11420 %}
11421 ins_pipe(ialu_reg_reg_alu0);
11422 %}
11423
11424 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
11425 %{
11426 match(Set dst (MulL src imm));
11427 effect(KILL cr);
11428
11429 ins_cost(300);
11430 format %{ "imulq $dst, $src, $imm\t# long" %}
11431 ins_encode %{
11432 __ imulq($dst$$Register, $src$$Register, $imm$$constant);
11433 %}
11434 ins_pipe(ialu_reg_reg_alu0);
11435 %}
11436
11437 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
11438 %{
11439 match(Set dst (MulL dst (LoadL src)));
11440 effect(KILL cr);
11441
11442 ins_cost(350);
11443 format %{ "imulq $dst, $src\t# long" %}
11444 ins_encode %{
11445 __ imulq($dst$$Register, $src$$Address);
11446 %}
11447 ins_pipe(ialu_reg_mem_alu0);
11448 %}
11449
11450
11451 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
11452 %{
11453 match(Set dst (MulL (LoadL src) imm));
11454 effect(KILL cr);
11455
11456 ins_cost(300);
11457 format %{ "imulq $dst, $src, $imm\t# long" %}
11458 ins_encode %{
11459 __ imulq($dst$$Register, $src$$Address, $imm$$constant);
11460 %}
11461 ins_pipe(ialu_reg_mem_alu0);
11462 %}
11463
11464 instruct mulHiLoL_rReg(rax_RegL rax, rdx_RegL rdx, rRegL src, rFlagsReg cr)
11465 %{
11466 match(MulHiLoL src rax);
11467 match(MulHiLoL rax src);
11468 effect(KILL cr);
11469
11470 ins_cost(300);
11471 format %{ "imulq RDX:RAX, RAX, $src\t# mulhilo" %}
11472 ins_encode %{
11473 __ imulq($src$$Register);
11474 %}
11475 ins_pipe(ialu_reg_reg_alu0);
11476 %}
11477
11478 instruct umulHiLoL_rReg(rax_RegL rax, rdx_RegL rdx, rRegL src, rFlagsReg cr)
11479 %{
11480 match(UMulHiLoL src rax);
11481 match(UMulHiLoL rax src);
11482 effect(KILL cr);
11483
11484 ins_cost(300);
11485 format %{ "mulq RDX:RAX, RAX, $src\t# umulhilo" %}
11486 ins_encode %{
11487 __ mulq($src$$Register);
11488 %}
11489 ins_pipe(ialu_reg_reg_alu0);
11490 %}
11491
11492 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
11493 %{
11494 match(Set dst (MulHiL src rax));
11495 effect(USE_KILL rax, KILL cr);
11496
11497 ins_cost(300);
11498 format %{ "imulq RDX:RAX, RAX, $src\t# mulhi" %}
11499 ins_encode %{
11500 __ imulq($src$$Register);
11501 %}
11502 ins_pipe(ialu_reg_reg_alu0);
11503 %}
11504
11505 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
11506 %{
11507 match(Set dst (UMulHiL src rax));
11508 effect(USE_KILL rax, KILL cr);
11509
11510 ins_cost(300);
11511 format %{ "mulq RDX:RAX, RAX, $src\t# umulhi" %}
11512 ins_encode %{
11513 __ mulq($src$$Register);
11514 %}
11515 ins_pipe(ialu_reg_reg_alu0);
11516 %}
11517
11518 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
11519 rFlagsReg cr)
11520 %{
11521 match(Set rax (DivI rax div));
11522 effect(KILL rdx, KILL cr);
11523
11524 ins_cost(30*100+10*100); // XXX
11525 format %{ "cmpl rax, 0x80000000\t# idiv\n\t"
11526 "jne,s normal\n\t"
11527 "xorl rdx, rdx\n\t"
11528 "cmpl $div, -1\n\t"
11529 "je,s done\n"
11530 "normal: cdql\n\t"
11531 "idivl $div\n"
11532 "done:" %}
11533 ins_encode(cdql_enc(div));
11534 ins_pipe(ialu_reg_reg_alu0);
11535 %}
11536
11537 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
11538 rFlagsReg cr)
11539 %{
11540 match(Set rax (DivL rax div));
11541 effect(KILL rdx, KILL cr);
11542
11543 ins_cost(30*100+10*100); // XXX
11544 format %{ "movq rdx, 0x8000000000000000\t# ldiv\n\t"
11545 "cmpq rax, rdx\n\t"
11546 "jne,s normal\n\t"
11547 "xorl rdx, rdx\n\t"
11548 "cmpq $div, -1\n\t"
11549 "je,s done\n"
11550 "normal: cdqq\n\t"
11551 "idivq $div\n"
11552 "done:" %}
11553 ins_encode(cdqq_enc(div));
11554 ins_pipe(ialu_reg_reg_alu0);
11555 %}
11556
11557 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
11558 %{
11559 match(Set rax (UDivI rax div));
11560 effect(KILL rdx, KILL cr);
11561
11562 ins_cost(300);
11563 format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
11564 ins_encode %{
11565 __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
11566 %}
11567 ins_pipe(ialu_reg_reg_alu0);
11568 %}
11569
11570 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
11571 %{
11572 match(Set rax (UDivL rax div));
11573 effect(KILL rdx, KILL cr);
11574
11575 ins_cost(300);
11576 format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
11577 ins_encode %{
11578 __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
11579 %}
11580 ins_pipe(ialu_reg_reg_alu0);
11581 %}
11582
11583 // Integer DIVMOD with Register, both quotient and mod results
11584 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
11585 rFlagsReg cr)
11586 %{
11587 match(DivModI rax div);
11588 effect(KILL cr);
11589
11590 ins_cost(30*100+10*100); // XXX
11591 format %{ "cmpl rax, 0x80000000\t# idiv\n\t"
11592 "jne,s normal\n\t"
11593 "xorl rdx, rdx\n\t"
11594 "cmpl $div, -1\n\t"
11595 "je,s done\n"
11596 "normal: cdql\n\t"
11597 "idivl $div\n"
11598 "done:" %}
11599 ins_encode(cdql_enc(div));
11600 ins_pipe(pipe_slow);
11601 %}
11602
11603 // Long DIVMOD with Register, both quotient and mod results
11604 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
11605 rFlagsReg cr)
11606 %{
11607 match(DivModL rax div);
11608 effect(KILL cr);
11609
11610 ins_cost(30*100+10*100); // XXX
11611 format %{ "movq rdx, 0x8000000000000000\t# ldiv\n\t"
11612 "cmpq rax, rdx\n\t"
11613 "jne,s normal\n\t"
11614 "xorl rdx, rdx\n\t"
11615 "cmpq $div, -1\n\t"
11616 "je,s done\n"
11617 "normal: cdqq\n\t"
11618 "idivq $div\n"
11619 "done:" %}
11620 ins_encode(cdqq_enc(div));
11621 ins_pipe(pipe_slow);
11622 %}
11623
11624 // Unsigned integer DIVMOD with Register, both quotient and mod results
11625 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
11626 no_rax_rdx_RegI div, rFlagsReg cr)
11627 %{
11628 match(UDivModI rax div);
11629 effect(TEMP tmp, KILL cr);
11630
11631 ins_cost(300);
11632 format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
11633 "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
11634 %}
11635 ins_encode %{
11636 __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
11637 %}
11638 ins_pipe(pipe_slow);
11639 %}
11640
11641 // Unsigned long DIVMOD with Register, both quotient and mod results
11642 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
11643 no_rax_rdx_RegL div, rFlagsReg cr)
11644 %{
11645 match(UDivModL rax div);
11646 effect(TEMP tmp, KILL cr);
11647
11648 ins_cost(300);
11649 format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
11650 "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
11651 %}
11652 ins_encode %{
11653 __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
11654 %}
11655 ins_pipe(pipe_slow);
11656 %}
11657
11658 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
11659 rFlagsReg cr)
11660 %{
11661 match(Set rdx (ModI rax div));
11662 effect(KILL rax, KILL cr);
11663
11664 ins_cost(300); // XXX
11665 format %{ "cmpl rax, 0x80000000\t# irem\n\t"
11666 "jne,s normal\n\t"
11667 "xorl rdx, rdx\n\t"
11668 "cmpl $div, -1\n\t"
11669 "je,s done\n"
11670 "normal: cdql\n\t"
11671 "idivl $div\n"
11672 "done:" %}
11673 ins_encode(cdql_enc(div));
11674 ins_pipe(ialu_reg_reg_alu0);
11675 %}
11676
11677 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
11678 rFlagsReg cr)
11679 %{
11680 match(Set rdx (ModL rax div));
11681 effect(KILL rax, KILL cr);
11682
11683 ins_cost(300); // XXX
11684 format %{ "movq rdx, 0x8000000000000000\t# lrem\n\t"
11685 "cmpq rax, rdx\n\t"
11686 "jne,s normal\n\t"
11687 "xorl rdx, rdx\n\t"
11688 "cmpq $div, -1\n\t"
11689 "je,s done\n"
11690 "normal: cdqq\n\t"
11691 "idivq $div\n"
11692 "done:" %}
11693 ins_encode(cdqq_enc(div));
11694 ins_pipe(ialu_reg_reg_alu0);
11695 %}
11696
11697 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
11698 %{
11699 match(Set rdx (UModI rax div));
11700 effect(KILL rax, KILL cr);
11701
11702 ins_cost(300);
11703 format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
11704 ins_encode %{
11705 __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
11706 %}
11707 ins_pipe(ialu_reg_reg_alu0);
11708 %}
11709
11710 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
11711 %{
11712 match(Set rdx (UModL rax div));
11713 effect(KILL rax, KILL cr);
11714
11715 ins_cost(300);
11716 format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
11717 ins_encode %{
11718 __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
11719 %}
11720 ins_pipe(ialu_reg_reg_alu0);
11721 %}
11722
11723 // Integer Shift Instructions
11724 // Shift Left by one, two, three
11725 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
11726 %{
11727 predicate(!UseAPX);
11728 match(Set dst (LShiftI dst shift));
11729 effect(KILL cr);
11730
11731 format %{ "sall $dst, $shift" %}
11732 ins_encode %{
11733 __ sall($dst$$Register, $shift$$constant);
11734 %}
11735 ins_pipe(ialu_reg);
11736 %}
11737
11738 // Shift Left by one, two, three
11739 instruct salI_rReg_immI2_ndd(rRegI dst, rRegI src, immI2 shift, rFlagsReg cr)
11740 %{
11741 predicate(UseAPX);
11742 match(Set dst (LShiftI src shift));
11743 effect(KILL cr);
11744 flag(PD::Flag_ndd_demotable_opr1);
11745
11746 format %{ "esall $dst, $src, $shift\t# int(ndd)" %}
11747 ins_encode %{
11748 __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
11749 %}
11750 ins_pipe(ialu_reg);
11751 %}
11752
11753 // Shift Left by 8-bit immediate
11754 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
11755 %{
11756 predicate(!UseAPX);
11757 match(Set dst (LShiftI dst shift));
11758 effect(KILL cr);
11759
11760 format %{ "sall $dst, $shift" %}
11761 ins_encode %{
11762 __ sall($dst$$Register, $shift$$constant);
11763 %}
11764 ins_pipe(ialu_reg);
11765 %}
11766
11767 // Shift Left by 8-bit immediate
11768 instruct salI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
11769 %{
11770 predicate(UseAPX);
11771 match(Set dst (LShiftI src shift));
11772 effect(KILL cr);
11773 flag(PD::Flag_ndd_demotable_opr1);
11774
11775 format %{ "esall $dst, $src, $shift\t# int (ndd)" %}
11776 ins_encode %{
11777 __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
11778 %}
11779 ins_pipe(ialu_reg);
11780 %}
11781
11782 // Shift Left by 8-bit immediate
11783 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
11784 %{
11785 match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
11786 effect(KILL cr);
11787
11788 format %{ "sall $dst, $shift" %}
11789 ins_encode %{
11790 __ sall($dst$$Address, $shift$$constant);
11791 %}
11792 ins_pipe(ialu_mem_imm);
11793 %}
11794
11795 // Shift Left by variable
11796 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
11797 %{
11798 predicate(!VM_Version::supports_bmi2());
11799 match(Set dst (LShiftI dst shift));
11800 effect(KILL cr);
11801
11802 format %{ "sall $dst, $shift" %}
11803 ins_encode %{
11804 __ sall($dst$$Register);
11805 %}
11806 ins_pipe(ialu_reg_reg);
11807 %}
11808
11809 // Shift Left by variable
11810 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
11811 %{
11812 predicate(!VM_Version::supports_bmi2());
11813 match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
11814 effect(KILL cr);
11815
11816 format %{ "sall $dst, $shift" %}
11817 ins_encode %{
11818 __ sall($dst$$Address);
11819 %}
11820 ins_pipe(ialu_mem_reg);
11821 %}
11822
11823 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
11824 %{
11825 predicate(VM_Version::supports_bmi2());
11826 match(Set dst (LShiftI src shift));
11827
11828 format %{ "shlxl $dst, $src, $shift" %}
11829 ins_encode %{
11830 __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
11831 %}
11832 ins_pipe(ialu_reg_reg);
11833 %}
11834
11835 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
11836 %{
11837 predicate(VM_Version::supports_bmi2());
11838 match(Set dst (LShiftI (LoadI src) shift));
11839 ins_cost(175);
11840 format %{ "shlxl $dst, $src, $shift" %}
11841 ins_encode %{
11842 __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
11843 %}
11844 ins_pipe(ialu_reg_mem);
11845 %}
11846
11847 // Arithmetic Shift Right by 8-bit immediate
11848 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
11849 %{
11850 predicate(!UseAPX);
11851 match(Set dst (RShiftI dst shift));
11852 effect(KILL cr);
11853
11854 format %{ "sarl $dst, $shift" %}
11855 ins_encode %{
11856 __ sarl($dst$$Register, $shift$$constant);
11857 %}
11858 ins_pipe(ialu_mem_imm);
11859 %}
11860
11861 // Arithmetic Shift Right by 8-bit immediate
11862 instruct sarI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
11863 %{
11864 predicate(UseAPX);
11865 match(Set dst (RShiftI src shift));
11866 effect(KILL cr);
11867 flag(PD::Flag_ndd_demotable_opr1);
11868
11869 format %{ "esarl $dst, $src, $shift\t# int (ndd)" %}
11870 ins_encode %{
11871 __ esarl($dst$$Register, $src$$Register, $shift$$constant, false);
11872 %}
11873 ins_pipe(ialu_mem_imm);
11874 %}
11875
11876 // Arithmetic Shift Right by 8-bit immediate
11877 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
11878 %{
11879 match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
11880 effect(KILL cr);
11881
11882 format %{ "sarl $dst, $shift" %}
11883 ins_encode %{
11884 __ sarl($dst$$Address, $shift$$constant);
11885 %}
11886 ins_pipe(ialu_mem_imm);
11887 %}
11888
11889 // Arithmetic Shift Right by variable
11890 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
11891 %{
11892 predicate(!VM_Version::supports_bmi2());
11893 match(Set dst (RShiftI dst shift));
11894 effect(KILL cr);
11895
11896 format %{ "sarl $dst, $shift" %}
11897 ins_encode %{
11898 __ sarl($dst$$Register);
11899 %}
11900 ins_pipe(ialu_reg_reg);
11901 %}
11902
11903 // Arithmetic Shift Right by variable
11904 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
11905 %{
11906 predicate(!VM_Version::supports_bmi2());
11907 match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
11908 effect(KILL cr);
11909
11910 format %{ "sarl $dst, $shift" %}
11911 ins_encode %{
11912 __ sarl($dst$$Address);
11913 %}
11914 ins_pipe(ialu_mem_reg);
11915 %}
11916
11917 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
11918 %{
11919 predicate(VM_Version::supports_bmi2());
11920 match(Set dst (RShiftI src shift));
11921
11922 format %{ "sarxl $dst, $src, $shift" %}
11923 ins_encode %{
11924 __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
11925 %}
11926 ins_pipe(ialu_reg_reg);
11927 %}
11928
11929 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
11930 %{
11931 predicate(VM_Version::supports_bmi2());
11932 match(Set dst (RShiftI (LoadI src) shift));
11933 ins_cost(175);
11934 format %{ "sarxl $dst, $src, $shift" %}
11935 ins_encode %{
11936 __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
11937 %}
11938 ins_pipe(ialu_reg_mem);
11939 %}
11940
11941 // Logical Shift Right by 8-bit immediate
11942 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
11943 %{
11944 predicate(!UseAPX);
11945 match(Set dst (URShiftI dst shift));
11946 effect(KILL cr);
11947
11948 format %{ "shrl $dst, $shift" %}
11949 ins_encode %{
11950 __ shrl($dst$$Register, $shift$$constant);
11951 %}
11952 ins_pipe(ialu_reg);
11953 %}
11954
11955 // Logical Shift Right by 8-bit immediate
11956 instruct shrI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
11957 %{
11958 predicate(UseAPX);
11959 match(Set dst (URShiftI src shift));
11960 effect(KILL cr);
11961 flag(PD::Flag_ndd_demotable_opr1);
11962
11963 format %{ "eshrl $dst, $src, $shift\t # int (ndd)" %}
11964 ins_encode %{
11965 __ eshrl($dst$$Register, $src$$Register, $shift$$constant, false);
11966 %}
11967 ins_pipe(ialu_reg);
11968 %}
11969
11970 // Logical Shift Right by 8-bit immediate
11971 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
11972 %{
11973 match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
11974 effect(KILL cr);
11975
11976 format %{ "shrl $dst, $shift" %}
11977 ins_encode %{
11978 __ shrl($dst$$Address, $shift$$constant);
11979 %}
11980 ins_pipe(ialu_mem_imm);
11981 %}
11982
11983 // Logical Shift Right by variable
11984 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
11985 %{
11986 predicate(!VM_Version::supports_bmi2());
11987 match(Set dst (URShiftI dst shift));
11988 effect(KILL cr);
11989
11990 format %{ "shrl $dst, $shift" %}
11991 ins_encode %{
11992 __ shrl($dst$$Register);
11993 %}
11994 ins_pipe(ialu_reg_reg);
11995 %}
11996
11997 // Logical Shift Right by variable
11998 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
11999 %{
12000 predicate(!VM_Version::supports_bmi2());
12001 match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
12002 effect(KILL cr);
12003
12004 format %{ "shrl $dst, $shift" %}
12005 ins_encode %{
12006 __ shrl($dst$$Address);
12007 %}
12008 ins_pipe(ialu_mem_reg);
12009 %}
12010
12011 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
12012 %{
12013 predicate(VM_Version::supports_bmi2());
12014 match(Set dst (URShiftI src shift));
12015
12016 format %{ "shrxl $dst, $src, $shift" %}
12017 ins_encode %{
12018 __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
12019 %}
12020 ins_pipe(ialu_reg_reg);
12021 %}
12022
12023 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
12024 %{
12025 predicate(VM_Version::supports_bmi2());
12026 match(Set dst (URShiftI (LoadI src) shift));
12027 ins_cost(175);
12028 format %{ "shrxl $dst, $src, $shift" %}
12029 ins_encode %{
12030 __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
12031 %}
12032 ins_pipe(ialu_reg_mem);
12033 %}
12034
12035 // Long Shift Instructions
12036 // Shift Left by one, two, three
12037 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
12038 %{
12039 predicate(!UseAPX);
12040 match(Set dst (LShiftL dst shift));
12041 effect(KILL cr);
12042
12043 format %{ "salq $dst, $shift" %}
12044 ins_encode %{
12045 __ salq($dst$$Register, $shift$$constant);
12046 %}
12047 ins_pipe(ialu_reg);
12048 %}
12049
12050 // Shift Left by one, two, three
12051 instruct salL_rReg_immI2_ndd(rRegL dst, rRegL src, immI2 shift, rFlagsReg cr)
12052 %{
12053 predicate(UseAPX);
12054 match(Set dst (LShiftL src shift));
12055 effect(KILL cr);
12056 flag(PD::Flag_ndd_demotable_opr1);
12057
12058 format %{ "esalq $dst, $src, $shift\t# long (ndd)" %}
12059 ins_encode %{
12060 __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
12061 %}
12062 ins_pipe(ialu_reg);
12063 %}
12064
12065 // Shift Left by 8-bit immediate
12066 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
12067 %{
12068 predicate(!UseAPX);
12069 match(Set dst (LShiftL dst shift));
12070 effect(KILL cr);
12071
12072 format %{ "salq $dst, $shift" %}
12073 ins_encode %{
12074 __ salq($dst$$Register, $shift$$constant);
12075 %}
12076 ins_pipe(ialu_reg);
12077 %}
12078
12079 // Shift Left by 8-bit immediate
12080 instruct salL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
12081 %{
12082 predicate(UseAPX);
12083 match(Set dst (LShiftL src shift));
12084 effect(KILL cr);
12085 flag(PD::Flag_ndd_demotable_opr1);
12086
12087 format %{ "esalq $dst, $src, $shift\t# long (ndd)" %}
12088 ins_encode %{
12089 __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
12090 %}
12091 ins_pipe(ialu_reg);
12092 %}
12093
12094 // Shift Left by 8-bit immediate
12095 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
12096 %{
12097 match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
12098 effect(KILL cr);
12099
12100 format %{ "salq $dst, $shift" %}
12101 ins_encode %{
12102 __ salq($dst$$Address, $shift$$constant);
12103 %}
12104 ins_pipe(ialu_mem_imm);
12105 %}
12106
12107 // Shift Left by variable
12108 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12109 %{
12110 predicate(!VM_Version::supports_bmi2());
12111 match(Set dst (LShiftL dst shift));
12112 effect(KILL cr);
12113
12114 format %{ "salq $dst, $shift" %}
12115 ins_encode %{
12116 __ salq($dst$$Register);
12117 %}
12118 ins_pipe(ialu_reg_reg);
12119 %}
12120
12121 // Shift Left by variable
12122 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
12123 %{
12124 predicate(!VM_Version::supports_bmi2());
12125 match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
12126 effect(KILL cr);
12127
12128 format %{ "salq $dst, $shift" %}
12129 ins_encode %{
12130 __ salq($dst$$Address);
12131 %}
12132 ins_pipe(ialu_mem_reg);
12133 %}
12134
12135 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
12136 %{
12137 predicate(VM_Version::supports_bmi2());
12138 match(Set dst (LShiftL src shift));
12139
12140 format %{ "shlxq $dst, $src, $shift" %}
12141 ins_encode %{
12142 __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
12143 %}
12144 ins_pipe(ialu_reg_reg);
12145 %}
12146
12147 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
12148 %{
12149 predicate(VM_Version::supports_bmi2());
12150 match(Set dst (LShiftL (LoadL src) shift));
12151 ins_cost(175);
12152 format %{ "shlxq $dst, $src, $shift" %}
12153 ins_encode %{
12154 __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
12155 %}
12156 ins_pipe(ialu_reg_mem);
12157 %}
12158
12159 // Arithmetic Shift Right by 8-bit immediate
12160 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
12161 %{
12162 predicate(!UseAPX);
12163 match(Set dst (RShiftL dst shift));
12164 effect(KILL cr);
12165
12166 format %{ "sarq $dst, $shift" %}
12167 ins_encode %{
12168 __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
12169 %}
12170 ins_pipe(ialu_mem_imm);
12171 %}
12172
12173 // Arithmetic Shift Right by 8-bit immediate
12174 instruct sarL_rReg_imm_ndd(rRegL dst, rRegL src, immI shift, rFlagsReg cr)
12175 %{
12176 predicate(UseAPX);
12177 match(Set dst (RShiftL src shift));
12178 effect(KILL cr);
12179 flag(PD::Flag_ndd_demotable_opr1);
12180
12181 format %{ "esarq $dst, $src, $shift\t# long (ndd)" %}
12182 ins_encode %{
12183 __ esarq($dst$$Register, $src$$Register, (unsigned char)($shift$$constant & 0x3F), false);
12184 %}
12185 ins_pipe(ialu_mem_imm);
12186 %}
12187
12188 // Arithmetic Shift Right by 8-bit immediate
12189 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
12190 %{
12191 match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
12192 effect(KILL cr);
12193
12194 format %{ "sarq $dst, $shift" %}
12195 ins_encode %{
12196 __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
12197 %}
12198 ins_pipe(ialu_mem_imm);
12199 %}
12200
12201 // Arithmetic Shift Right by variable
12202 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12203 %{
12204 predicate(!VM_Version::supports_bmi2());
12205 match(Set dst (RShiftL dst shift));
12206 effect(KILL cr);
12207
12208 format %{ "sarq $dst, $shift" %}
12209 ins_encode %{
12210 __ sarq($dst$$Register);
12211 %}
12212 ins_pipe(ialu_reg_reg);
12213 %}
12214
12215 // Arithmetic Shift Right by variable
12216 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
12217 %{
12218 predicate(!VM_Version::supports_bmi2());
12219 match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
12220 effect(KILL cr);
12221
12222 format %{ "sarq $dst, $shift" %}
12223 ins_encode %{
12224 __ sarq($dst$$Address);
12225 %}
12226 ins_pipe(ialu_mem_reg);
12227 %}
12228
12229 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
12230 %{
12231 predicate(VM_Version::supports_bmi2());
12232 match(Set dst (RShiftL src shift));
12233
12234 format %{ "sarxq $dst, $src, $shift" %}
12235 ins_encode %{
12236 __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
12237 %}
12238 ins_pipe(ialu_reg_reg);
12239 %}
12240
12241 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
12242 %{
12243 predicate(VM_Version::supports_bmi2());
12244 match(Set dst (RShiftL (LoadL src) shift));
12245 ins_cost(175);
12246 format %{ "sarxq $dst, $src, $shift" %}
12247 ins_encode %{
12248 __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
12249 %}
12250 ins_pipe(ialu_reg_mem);
12251 %}
12252
12253 // Logical Shift Right by 8-bit immediate
12254 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
12255 %{
12256 predicate(!UseAPX);
12257 match(Set dst (URShiftL dst shift));
12258 effect(KILL cr);
12259
12260 format %{ "shrq $dst, $shift" %}
12261 ins_encode %{
12262 __ shrq($dst$$Register, $shift$$constant);
12263 %}
12264 ins_pipe(ialu_reg);
12265 %}
12266
12267 // Logical Shift Right by 8-bit immediate
12268 instruct shrL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
12269 %{
12270 predicate(UseAPX);
12271 match(Set dst (URShiftL src shift));
12272 effect(KILL cr);
12273 flag(PD::Flag_ndd_demotable_opr1);
12274
12275 format %{ "eshrq $dst, $src, $shift\t# long (ndd)" %}
12276 ins_encode %{
12277 __ eshrq($dst$$Register, $src$$Register, $shift$$constant, false);
12278 %}
12279 ins_pipe(ialu_reg);
12280 %}
12281
12282 // Logical Shift Right by 8-bit immediate
12283 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
12284 %{
12285 match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
12286 effect(KILL cr);
12287
12288 format %{ "shrq $dst, $shift" %}
12289 ins_encode %{
12290 __ shrq($dst$$Address, $shift$$constant);
12291 %}
12292 ins_pipe(ialu_mem_imm);
12293 %}
12294
12295 // Logical Shift Right by variable
12296 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12297 %{
12298 predicate(!VM_Version::supports_bmi2());
12299 match(Set dst (URShiftL dst shift));
12300 effect(KILL cr);
12301
12302 format %{ "shrq $dst, $shift" %}
12303 ins_encode %{
12304 __ shrq($dst$$Register);
12305 %}
12306 ins_pipe(ialu_reg_reg);
12307 %}
12308
12309 // Logical Shift Right by variable
12310 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
12311 %{
12312 predicate(!VM_Version::supports_bmi2());
12313 match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
12314 effect(KILL cr);
12315
12316 format %{ "shrq $dst, $shift" %}
12317 ins_encode %{
12318 __ shrq($dst$$Address);
12319 %}
12320 ins_pipe(ialu_mem_reg);
12321 %}
12322
12323 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
12324 %{
12325 predicate(VM_Version::supports_bmi2());
12326 match(Set dst (URShiftL src shift));
12327
12328 format %{ "shrxq $dst, $src, $shift" %}
12329 ins_encode %{
12330 __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
12331 %}
12332 ins_pipe(ialu_reg_reg);
12333 %}
12334
12335 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
12336 %{
12337 predicate(VM_Version::supports_bmi2());
12338 match(Set dst (URShiftL (LoadL src) shift));
12339 ins_cost(175);
12340 format %{ "shrxq $dst, $src, $shift" %}
12341 ins_encode %{
12342 __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
12343 %}
12344 ins_pipe(ialu_reg_mem);
12345 %}
12346
12347 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
12348 // This idiom is used by the compiler for the i2b bytecode.
12349 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
12350 %{
12351 match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
12352
12353 format %{ "movsbl $dst, $src\t# i2b" %}
12354 ins_encode %{
12355 __ movsbl($dst$$Register, $src$$Register);
12356 %}
12357 ins_pipe(ialu_reg_reg);
12358 %}
12359
12360 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
12361 // This idiom is used by the compiler the i2s bytecode.
12362 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
12363 %{
12364 match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
12365
12366 format %{ "movswl $dst, $src\t# i2s" %}
12367 ins_encode %{
12368 __ movswl($dst$$Register, $src$$Register);
12369 %}
12370 ins_pipe(ialu_reg_reg);
12371 %}
12372
12373 // ROL/ROR instructions
12374
12375 // Rotate left by constant.
12376 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
12377 %{
12378 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12379 match(Set dst (RotateLeft dst shift));
12380 effect(KILL cr);
12381 format %{ "roll $dst, $shift" %}
12382 ins_encode %{
12383 __ roll($dst$$Register, $shift$$constant);
12384 %}
12385 ins_pipe(ialu_reg);
12386 %}
12387
12388 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
12389 %{
12390 predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12391 match(Set dst (RotateLeft src shift));
12392 format %{ "rolxl $dst, $src, $shift" %}
12393 ins_encode %{
12394 int shift = 32 - ($shift$$constant & 31);
12395 __ rorxl($dst$$Register, $src$$Register, shift);
12396 %}
12397 ins_pipe(ialu_reg_reg);
12398 %}
12399
12400 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
12401 %{
12402 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12403 match(Set dst (RotateLeft (LoadI src) shift));
12404 ins_cost(175);
12405 format %{ "rolxl $dst, $src, $shift" %}
12406 ins_encode %{
12407 int shift = 32 - ($shift$$constant & 31);
12408 __ rorxl($dst$$Register, $src$$Address, shift);
12409 %}
12410 ins_pipe(ialu_reg_mem);
12411 %}
12412
12413 // Rotate Left by variable
12414 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
12415 %{
12416 predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
12417 match(Set dst (RotateLeft dst shift));
12418 effect(KILL cr);
12419 format %{ "roll $dst, $shift" %}
12420 ins_encode %{
12421 __ roll($dst$$Register);
12422 %}
12423 ins_pipe(ialu_reg_reg);
12424 %}
12425
12426 // Rotate Left by variable
12427 instruct rolI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
12428 %{
12429 predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
12430 match(Set dst (RotateLeft src shift));
12431 effect(KILL cr);
12432 flag(PD::Flag_ndd_demotable_opr1);
12433
12434 format %{ "eroll $dst, $src, $shift\t# rotate left (int ndd)" %}
12435 ins_encode %{
12436 __ eroll($dst$$Register, $src$$Register, false);
12437 %}
12438 ins_pipe(ialu_reg_reg);
12439 %}
12440
12441 // Rotate Right by constant.
12442 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
12443 %{
12444 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12445 match(Set dst (RotateRight dst shift));
12446 effect(KILL cr);
12447 format %{ "rorl $dst, $shift" %}
12448 ins_encode %{
12449 __ rorl($dst$$Register, $shift$$constant);
12450 %}
12451 ins_pipe(ialu_reg);
12452 %}
12453
12454 // Rotate Right by constant.
12455 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
12456 %{
12457 predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12458 match(Set dst (RotateRight src shift));
12459 format %{ "rorxl $dst, $src, $shift" %}
12460 ins_encode %{
12461 __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
12462 %}
12463 ins_pipe(ialu_reg_reg);
12464 %}
12465
12466 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
12467 %{
12468 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12469 match(Set dst (RotateRight (LoadI src) shift));
12470 ins_cost(175);
12471 format %{ "rorxl $dst, $src, $shift" %}
12472 ins_encode %{
12473 __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
12474 %}
12475 ins_pipe(ialu_reg_mem);
12476 %}
12477
12478 // Rotate Right by variable
12479 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
12480 %{
12481 predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
12482 match(Set dst (RotateRight dst shift));
12483 effect(KILL cr);
12484 format %{ "rorl $dst, $shift" %}
12485 ins_encode %{
12486 __ rorl($dst$$Register);
12487 %}
12488 ins_pipe(ialu_reg_reg);
12489 %}
12490
12491 // Rotate Right by variable
12492 instruct rorI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
12493 %{
12494 predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
12495 match(Set dst (RotateRight src shift));
12496 effect(KILL cr);
12497 flag(PD::Flag_ndd_demotable_opr1);
12498
12499 format %{ "erorl $dst, $src, $shift\t# rotate right(int ndd)" %}
12500 ins_encode %{
12501 __ erorl($dst$$Register, $src$$Register, false);
12502 %}
12503 ins_pipe(ialu_reg_reg);
12504 %}
12505
12506 // Rotate Left by constant.
12507 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
12508 %{
12509 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12510 match(Set dst (RotateLeft dst shift));
12511 effect(KILL cr);
12512 format %{ "rolq $dst, $shift" %}
12513 ins_encode %{
12514 __ rolq($dst$$Register, $shift$$constant);
12515 %}
12516 ins_pipe(ialu_reg);
12517 %}
12518
12519 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
12520 %{
12521 predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12522 match(Set dst (RotateLeft src shift));
12523 format %{ "rolxq $dst, $src, $shift" %}
12524 ins_encode %{
12525 int shift = 64 - ($shift$$constant & 63);
12526 __ rorxq($dst$$Register, $src$$Register, shift);
12527 %}
12528 ins_pipe(ialu_reg_reg);
12529 %}
12530
12531 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
12532 %{
12533 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12534 match(Set dst (RotateLeft (LoadL src) shift));
12535 ins_cost(175);
12536 format %{ "rolxq $dst, $src, $shift" %}
12537 ins_encode %{
12538 int shift = 64 - ($shift$$constant & 63);
12539 __ rorxq($dst$$Register, $src$$Address, shift);
12540 %}
12541 ins_pipe(ialu_reg_mem);
12542 %}
12543
12544 // Rotate Left by variable
12545 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12546 %{
12547 predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
12548 match(Set dst (RotateLeft dst shift));
12549 effect(KILL cr);
12550
12551 format %{ "rolq $dst, $shift" %}
12552 ins_encode %{
12553 __ rolq($dst$$Register);
12554 %}
12555 ins_pipe(ialu_reg_reg);
12556 %}
12557
12558 // Rotate Left by variable
12559 instruct rolL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
12560 %{
12561 predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
12562 match(Set dst (RotateLeft src shift));
12563 effect(KILL cr);
12564 flag(PD::Flag_ndd_demotable_opr1);
12565
12566 format %{ "erolq $dst, $src, $shift\t# rotate left(long ndd)" %}
12567 ins_encode %{
12568 __ erolq($dst$$Register, $src$$Register, false);
12569 %}
12570 ins_pipe(ialu_reg_reg);
12571 %}
12572
12573 // Rotate Right by constant.
12574 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
12575 %{
12576 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12577 match(Set dst (RotateRight dst shift));
12578 effect(KILL cr);
12579 format %{ "rorq $dst, $shift" %}
12580 ins_encode %{
12581 __ rorq($dst$$Register, $shift$$constant);
12582 %}
12583 ins_pipe(ialu_reg);
12584 %}
12585
12586 // Rotate Right by constant
12587 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
12588 %{
12589 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12590 match(Set dst (RotateRight src shift));
12591 format %{ "rorxq $dst, $src, $shift" %}
12592 ins_encode %{
12593 __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
12594 %}
12595 ins_pipe(ialu_reg_reg);
12596 %}
12597
12598 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
12599 %{
12600 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12601 match(Set dst (RotateRight (LoadL src) shift));
12602 ins_cost(175);
12603 format %{ "rorxq $dst, $src, $shift" %}
12604 ins_encode %{
12605 __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
12606 %}
12607 ins_pipe(ialu_reg_mem);
12608 %}
12609
12610 // Rotate Right by variable
12611 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12612 %{
12613 predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
12614 match(Set dst (RotateRight dst shift));
12615 effect(KILL cr);
12616 format %{ "rorq $dst, $shift" %}
12617 ins_encode %{
12618 __ rorq($dst$$Register);
12619 %}
12620 ins_pipe(ialu_reg_reg);
12621 %}
12622
12623 // Rotate Right by variable
12624 instruct rorL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
12625 %{
12626 predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
12627 match(Set dst (RotateRight src shift));
12628 effect(KILL cr);
12629 flag(PD::Flag_ndd_demotable_opr1);
12630
12631 format %{ "erorq $dst, $src, $shift\t# rotate right(long ndd)" %}
12632 ins_encode %{
12633 __ erorq($dst$$Register, $src$$Register, false);
12634 %}
12635 ins_pipe(ialu_reg_reg);
12636 %}
12637
12638 //----------------------------- CompressBits/ExpandBits ------------------------
12639
12640 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
12641 predicate(n->bottom_type()->isa_long());
12642 match(Set dst (CompressBits src mask));
12643 format %{ "pextq $dst, $src, $mask\t! parallel bit extract" %}
12644 ins_encode %{
12645 __ pextq($dst$$Register, $src$$Register, $mask$$Register);
12646 %}
12647 ins_pipe( pipe_slow );
12648 %}
12649
12650 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
12651 predicate(n->bottom_type()->isa_long());
12652 match(Set dst (ExpandBits src mask));
12653 format %{ "pdepq $dst, $src, $mask\t! parallel bit deposit" %}
12654 ins_encode %{
12655 __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
12656 %}
12657 ins_pipe( pipe_slow );
12658 %}
12659
12660 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
12661 predicate(n->bottom_type()->isa_long());
12662 match(Set dst (CompressBits src (LoadL mask)));
12663 format %{ "pextq $dst, $src, $mask\t! parallel bit extract" %}
12664 ins_encode %{
12665 __ pextq($dst$$Register, $src$$Register, $mask$$Address);
12666 %}
12667 ins_pipe( pipe_slow );
12668 %}
12669
12670 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
12671 predicate(n->bottom_type()->isa_long());
12672 match(Set dst (ExpandBits src (LoadL mask)));
12673 format %{ "pdepq $dst, $src, $mask\t! parallel bit deposit" %}
12674 ins_encode %{
12675 __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
12676 %}
12677 ins_pipe( pipe_slow );
12678 %}
12679
12680
12681 // Logical Instructions
12682
12683 // Integer Logical Instructions
12684
12685 // And Instructions
12686 // And Register with Register
12687 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
12688 %{
12689 predicate(!UseAPX);
12690 match(Set dst (AndI dst src));
12691 effect(KILL cr);
12692 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12693
12694 format %{ "andl $dst, $src\t# int" %}
12695 ins_encode %{
12696 __ andl($dst$$Register, $src$$Register);
12697 %}
12698 ins_pipe(ialu_reg_reg);
12699 %}
12700
12701 // And Register with Register using New Data Destination (NDD)
12702 instruct andI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
12703 %{
12704 predicate(UseAPX);
12705 match(Set dst (AndI src1 src2));
12706 effect(KILL cr);
12707 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
12708
12709 format %{ "eandl $dst, $src1, $src2\t# int ndd" %}
12710 ins_encode %{
12711 __ eandl($dst$$Register, $src1$$Register, $src2$$Register, false);
12712
12713 %}
12714 ins_pipe(ialu_reg_reg);
12715 %}
12716
12717 // And Register with Immediate 255
12718 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
12719 %{
12720 match(Set dst (AndI src mask));
12721
12722 format %{ "movzbl $dst, $src\t# int & 0xFF" %}
12723 ins_encode %{
12724 __ movzbl($dst$$Register, $src$$Register);
12725 %}
12726 ins_pipe(ialu_reg);
12727 %}
12728
12729 // And Register with Immediate 255 and promote to long
12730 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
12731 %{
12732 match(Set dst (ConvI2L (AndI src mask)));
12733
12734 format %{ "movzbl $dst, $src\t# int & 0xFF -> long" %}
12735 ins_encode %{
12736 __ movzbl($dst$$Register, $src$$Register);
12737 %}
12738 ins_pipe(ialu_reg);
12739 %}
12740
12741 // And Register with Immediate 65535
12742 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
12743 %{
12744 match(Set dst (AndI src mask));
12745
12746 format %{ "movzwl $dst, $src\t# int & 0xFFFF" %}
12747 ins_encode %{
12748 __ movzwl($dst$$Register, $src$$Register);
12749 %}
12750 ins_pipe(ialu_reg);
12751 %}
12752
12753 // And Register with Immediate 65535 and promote to long
12754 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
12755 %{
12756 match(Set dst (ConvI2L (AndI src mask)));
12757
12758 format %{ "movzwl $dst, $src\t# int & 0xFFFF -> long" %}
12759 ins_encode %{
12760 __ movzwl($dst$$Register, $src$$Register);
12761 %}
12762 ins_pipe(ialu_reg);
12763 %}
12764
12765 // Can skip int2long conversions after AND with small bitmask
12766 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src, immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
12767 %{
12768 predicate(VM_Version::supports_bmi2());
12769 ins_cost(125);
12770 effect(TEMP tmp, KILL cr);
12771 match(Set dst (ConvI2L (AndI src mask)));
12772 format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int & immI_Pow2M1 -> long" %}
12773 ins_encode %{
12774 __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
12775 __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
12776 %}
12777 ins_pipe(ialu_reg_reg);
12778 %}
12779
12780 // And Register with Immediate
12781 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
12782 %{
12783 predicate(!UseAPX);
12784 match(Set dst (AndI dst src));
12785 effect(KILL cr);
12786 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12787
12788 format %{ "andl $dst, $src\t# int" %}
12789 ins_encode %{
12790 __ andl($dst$$Register, $src$$constant);
12791 %}
12792 ins_pipe(ialu_reg);
12793 %}
12794
12795 instruct andI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
12796 %{
12797 predicate(UseAPX);
12798 match(Set dst (AndI src1 src2));
12799 effect(KILL cr);
12800 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
12801
12802 format %{ "eandl $dst, $src1, $src2\t# int ndd" %}
12803 ins_encode %{
12804 __ eandl($dst$$Register, $src1$$Register, $src2$$constant, false);
12805 %}
12806 ins_pipe(ialu_reg);
12807 %}
12808
12809 // And Register with Memory
12810 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
12811 %{
12812 match(Set dst (AndI dst (LoadI src)));
12813 effect(KILL cr);
12814 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12815
12816 ins_cost(150);
12817 format %{ "andl $dst, $src\t# int" %}
12818 ins_encode %{
12819 __ andl($dst$$Register, $src$$Address);
12820 %}
12821 ins_pipe(ialu_reg_mem);
12822 %}
12823
12824 // And Memory with Register
12825 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
12826 %{
12827 match(Set dst (StoreB dst (AndI (LoadB dst) src)));
12828 effect(KILL cr);
12829 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12830
12831 ins_cost(150);
12832 format %{ "andb $dst, $src\t# byte" %}
12833 ins_encode %{
12834 __ andb($dst$$Address, $src$$Register);
12835 %}
12836 ins_pipe(ialu_mem_reg);
12837 %}
12838
12839 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
12840 %{
12841 match(Set dst (StoreI dst (AndI (LoadI dst) src)));
12842 effect(KILL cr);
12843 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12844
12845 ins_cost(150);
12846 format %{ "andl $dst, $src\t# int" %}
12847 ins_encode %{
12848 __ andl($dst$$Address, $src$$Register);
12849 %}
12850 ins_pipe(ialu_mem_reg);
12851 %}
12852
12853 // And Memory with Immediate
12854 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
12855 %{
12856 match(Set dst (StoreI dst (AndI (LoadI dst) src)));
12857 effect(KILL cr);
12858 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12859
12860 ins_cost(125);
12861 format %{ "andl $dst, $src\t# int" %}
12862 ins_encode %{
12863 __ andl($dst$$Address, $src$$constant);
12864 %}
12865 ins_pipe(ialu_mem_imm);
12866 %}
12867
12868 // BMI1 instructions
12869 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
12870 match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
12871 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12872 effect(KILL cr);
12873 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12874
12875 ins_cost(125);
12876 format %{ "andnl $dst, $src1, $src2" %}
12877
12878 ins_encode %{
12879 __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
12880 %}
12881 ins_pipe(ialu_reg_mem);
12882 %}
12883
12884 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
12885 match(Set dst (AndI (XorI src1 minus_1) src2));
12886 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12887 effect(KILL cr);
12888 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12889
12890 format %{ "andnl $dst, $src1, $src2" %}
12891
12892 ins_encode %{
12893 __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
12894 %}
12895 ins_pipe(ialu_reg);
12896 %}
12897
12898 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
12899 match(Set dst (AndI (SubI imm_zero src) src));
12900 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12901 effect(KILL cr);
12902 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12903
12904 format %{ "blsil $dst, $src" %}
12905
12906 ins_encode %{
12907 __ blsil($dst$$Register, $src$$Register);
12908 %}
12909 ins_pipe(ialu_reg);
12910 %}
12911
12912 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
12913 match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
12914 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12915 effect(KILL cr);
12916 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12917
12918 ins_cost(125);
12919 format %{ "blsil $dst, $src" %}
12920
12921 ins_encode %{
12922 __ blsil($dst$$Register, $src$$Address);
12923 %}
12924 ins_pipe(ialu_reg_mem);
12925 %}
12926
12927 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
12928 %{
12929 match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
12930 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12931 effect(KILL cr);
12932 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
12933
12934 ins_cost(125);
12935 format %{ "blsmskl $dst, $src" %}
12936
12937 ins_encode %{
12938 __ blsmskl($dst$$Register, $src$$Address);
12939 %}
12940 ins_pipe(ialu_reg_mem);
12941 %}
12942
12943 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
12944 %{
12945 match(Set dst (XorI (AddI src minus_1) src));
12946 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12947 effect(KILL cr);
12948 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
12949
12950 format %{ "blsmskl $dst, $src" %}
12951
12952 ins_encode %{
12953 __ blsmskl($dst$$Register, $src$$Register);
12954 %}
12955
12956 ins_pipe(ialu_reg);
12957 %}
12958
12959 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
12960 %{
12961 match(Set dst (AndI (AddI src minus_1) src) );
12962 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12963 effect(KILL cr);
12964 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12965
12966 format %{ "blsrl $dst, $src" %}
12967
12968 ins_encode %{
12969 __ blsrl($dst$$Register, $src$$Register);
12970 %}
12971
12972 ins_pipe(ialu_reg_mem);
12973 %}
12974
12975 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
12976 %{
12977 match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
12978 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12979 effect(KILL cr);
12980 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12981
12982 ins_cost(125);
12983 format %{ "blsrl $dst, $src" %}
12984
12985 ins_encode %{
12986 __ blsrl($dst$$Register, $src$$Address);
12987 %}
12988
12989 ins_pipe(ialu_reg);
12990 %}
12991
12992 // Or Instructions
12993 // Or Register with Register
12994 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
12995 %{
12996 predicate(!UseAPX);
12997 match(Set dst (OrI dst src));
12998 effect(KILL cr);
12999 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13000
13001 format %{ "orl $dst, $src\t# int" %}
13002 ins_encode %{
13003 __ orl($dst$$Register, $src$$Register);
13004 %}
13005 ins_pipe(ialu_reg_reg);
13006 %}
13007
13008 // Or Register with Register using New Data Destination (NDD)
13009 instruct orI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13010 %{
13011 predicate(UseAPX);
13012 match(Set dst (OrI src1 src2));
13013 effect(KILL cr);
13014 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13015
13016 format %{ "eorl $dst, $src1, $src2\t# int ndd" %}
13017 ins_encode %{
13018 __ eorl($dst$$Register, $src1$$Register, $src2$$Register, false);
13019 %}
13020 ins_pipe(ialu_reg_reg);
13021 %}
13022
13023 // Or Register with Immediate
13024 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
13025 %{
13026 predicate(!UseAPX);
13027 match(Set dst (OrI dst src));
13028 effect(KILL cr);
13029 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13030
13031 format %{ "orl $dst, $src\t# int" %}
13032 ins_encode %{
13033 __ orl($dst$$Register, $src$$constant);
13034 %}
13035 ins_pipe(ialu_reg);
13036 %}
13037
13038 instruct orI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
13039 %{
13040 predicate(UseAPX);
13041 match(Set dst (OrI src1 src2));
13042 effect(KILL cr);
13043 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13044
13045 format %{ "eorl $dst, $src1, $src2\t# int ndd" %}
13046 ins_encode %{
13047 __ eorl($dst$$Register, $src1$$Register, $src2$$constant, false);
13048 %}
13049 ins_pipe(ialu_reg);
13050 %}
13051
13052 instruct orI_rReg_imm_rReg_ndd(rRegI dst, immI src1, rRegI src2, rFlagsReg cr)
13053 %{
13054 predicate(UseAPX);
13055 match(Set dst (OrI src1 src2));
13056 effect(KILL cr);
13057 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13058
13059 format %{ "eorl $dst, $src2, $src1\t# int ndd" %}
13060 ins_encode %{
13061 __ eorl($dst$$Register, $src2$$Register, $src1$$constant, false);
13062 %}
13063 ins_pipe(ialu_reg);
13064 %}
13065
13066 // Or Register with Memory
13067 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
13068 %{
13069 match(Set dst (OrI dst (LoadI src)));
13070 effect(KILL cr);
13071 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13072
13073 ins_cost(150);
13074 format %{ "orl $dst, $src\t# int" %}
13075 ins_encode %{
13076 __ orl($dst$$Register, $src$$Address);
13077 %}
13078 ins_pipe(ialu_reg_mem);
13079 %}
13080
13081 // Or Memory with Register
13082 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
13083 %{
13084 match(Set dst (StoreB dst (OrI (LoadB dst) src)));
13085 effect(KILL cr);
13086 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13087
13088 ins_cost(150);
13089 format %{ "orb $dst, $src\t# byte" %}
13090 ins_encode %{
13091 __ orb($dst$$Address, $src$$Register);
13092 %}
13093 ins_pipe(ialu_mem_reg);
13094 %}
13095
13096 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
13097 %{
13098 match(Set dst (StoreI dst (OrI (LoadI dst) src)));
13099 effect(KILL cr);
13100 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13101
13102 ins_cost(150);
13103 format %{ "orl $dst, $src\t# int" %}
13104 ins_encode %{
13105 __ orl($dst$$Address, $src$$Register);
13106 %}
13107 ins_pipe(ialu_mem_reg);
13108 %}
13109
13110 // Or Memory with Immediate
13111 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
13112 %{
13113 match(Set dst (StoreI dst (OrI (LoadI dst) src)));
13114 effect(KILL cr);
13115 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13116
13117 ins_cost(125);
13118 format %{ "orl $dst, $src\t# int" %}
13119 ins_encode %{
13120 __ orl($dst$$Address, $src$$constant);
13121 %}
13122 ins_pipe(ialu_mem_imm);
13123 %}
13124
13125 // Xor Instructions
13126 // Xor Register with Register
13127 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
13128 %{
13129 predicate(!UseAPX);
13130 match(Set dst (XorI dst src));
13131 effect(KILL cr);
13132 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13133
13134 format %{ "xorl $dst, $src\t# int" %}
13135 ins_encode %{
13136 __ xorl($dst$$Register, $src$$Register);
13137 %}
13138 ins_pipe(ialu_reg_reg);
13139 %}
13140
13141 // Xor Register with Register using New Data Destination (NDD)
13142 instruct xorI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13143 %{
13144 predicate(UseAPX);
13145 match(Set dst (XorI src1 src2));
13146 effect(KILL cr);
13147 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13148
13149 format %{ "exorl $dst, $src1, $src2\t# int ndd" %}
13150 ins_encode %{
13151 __ exorl($dst$$Register, $src1$$Register, $src2$$Register, false);
13152 %}
13153 ins_pipe(ialu_reg_reg);
13154 %}
13155
13156 // Xor Register with Immediate -1
13157 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm)
13158 %{
13159 predicate(!UseAPX);
13160 match(Set dst (XorI dst imm));
13161
13162 format %{ "notl $dst" %}
13163 ins_encode %{
13164 __ notl($dst$$Register);
13165 %}
13166 ins_pipe(ialu_reg);
13167 %}
13168
13169 instruct xorI_rReg_im1_ndd(rRegI dst, rRegI src, immI_M1 imm)
13170 %{
13171 match(Set dst (XorI src imm));
13172 predicate(UseAPX);
13173 flag(PD::Flag_ndd_demotable_opr1);
13174
13175 format %{ "enotl $dst, $src" %}
13176 ins_encode %{
13177 __ enotl($dst$$Register, $src$$Register);
13178 %}
13179 ins_pipe(ialu_reg);
13180 %}
13181
13182 // Xor Register with Immediate
13183 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
13184 %{
13185 // Strict predicate check to make selection of xorI_rReg_im1 cost agnostic if immI src is -1.
13186 predicate(!UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
13187 match(Set dst (XorI dst src));
13188 effect(KILL cr);
13189 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13190
13191 format %{ "xorl $dst, $src\t# int" %}
13192 ins_encode %{
13193 __ xorl($dst$$Register, $src$$constant);
13194 %}
13195 ins_pipe(ialu_reg);
13196 %}
13197
13198 instruct xorI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
13199 %{
13200 // Strict predicate check to make selection of xorI_rReg_im1_ndd cost agnostic if immI src2 is -1.
13201 predicate(UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
13202 match(Set dst (XorI src1 src2));
13203 effect(KILL cr);
13204 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13205
13206 format %{ "exorl $dst, $src1, $src2\t# int ndd" %}
13207 ins_encode %{
13208 __ exorl($dst$$Register, $src1$$Register, $src2$$constant, false);
13209 %}
13210 ins_pipe(ialu_reg);
13211 %}
13212
13213 // Xor Register with Memory
13214 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
13215 %{
13216 match(Set dst (XorI dst (LoadI src)));
13217 effect(KILL cr);
13218 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13219
13220 ins_cost(150);
13221 format %{ "xorl $dst, $src\t# int" %}
13222 ins_encode %{
13223 __ xorl($dst$$Register, $src$$Address);
13224 %}
13225 ins_pipe(ialu_reg_mem);
13226 %}
13227
13228 // Xor Memory with Register
13229 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
13230 %{
13231 match(Set dst (StoreB dst (XorI (LoadB dst) src)));
13232 effect(KILL cr);
13233 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13234
13235 ins_cost(150);
13236 format %{ "xorb $dst, $src\t# byte" %}
13237 ins_encode %{
13238 __ xorb($dst$$Address, $src$$Register);
13239 %}
13240 ins_pipe(ialu_mem_reg);
13241 %}
13242
13243 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
13244 %{
13245 match(Set dst (StoreI dst (XorI (LoadI dst) src)));
13246 effect(KILL cr);
13247 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13248
13249 ins_cost(150);
13250 format %{ "xorl $dst, $src\t# int" %}
13251 ins_encode %{
13252 __ xorl($dst$$Address, $src$$Register);
13253 %}
13254 ins_pipe(ialu_mem_reg);
13255 %}
13256
13257 // Xor Memory with Immediate
13258 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
13259 %{
13260 match(Set dst (StoreI dst (XorI (LoadI dst) src)));
13261 effect(KILL cr);
13262 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13263
13264 ins_cost(125);
13265 format %{ "xorl $dst, $src\t# int" %}
13266 ins_encode %{
13267 __ xorl($dst$$Address, $src$$constant);
13268 %}
13269 ins_pipe(ialu_mem_imm);
13270 %}
13271
13272
13273 // Long Logical Instructions
13274
13275 // And Instructions
13276 // And Register with Register
13277 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
13278 %{
13279 predicate(!UseAPX);
13280 match(Set dst (AndL dst src));
13281 effect(KILL cr);
13282 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13283
13284 format %{ "andq $dst, $src\t# long" %}
13285 ins_encode %{
13286 __ andq($dst$$Register, $src$$Register);
13287 %}
13288 ins_pipe(ialu_reg_reg);
13289 %}
13290
13291 // And Register with Register using New Data Destination (NDD)
13292 instruct andL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
13293 %{
13294 predicate(UseAPX);
13295 match(Set dst (AndL src1 src2));
13296 effect(KILL cr);
13297 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13298
13299 format %{ "eandq $dst, $src1, $src2\t# long ndd" %}
13300 ins_encode %{
13301 __ eandq($dst$$Register, $src1$$Register, $src2$$Register, false);
13302
13303 %}
13304 ins_pipe(ialu_reg_reg);
13305 %}
13306
13307 // And Register with Immediate 255
13308 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
13309 %{
13310 match(Set dst (AndL src mask));
13311
13312 format %{ "movzbl $dst, $src\t# long & 0xFF" %}
13313 ins_encode %{
13314 // movzbl zeroes out the upper 32-bit and does not need REX.W
13315 __ movzbl($dst$$Register, $src$$Register);
13316 %}
13317 ins_pipe(ialu_reg);
13318 %}
13319
13320 // And Register with Immediate 65535
13321 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
13322 %{
13323 match(Set dst (AndL src mask));
13324
13325 format %{ "movzwl $dst, $src\t# long & 0xFFFF" %}
13326 ins_encode %{
13327 // movzwl zeroes out the upper 32-bit and does not need REX.W
13328 __ movzwl($dst$$Register, $src$$Register);
13329 %}
13330 ins_pipe(ialu_reg);
13331 %}
13332
13333 // And Register with Immediate
13334 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
13335 %{
13336 predicate(!UseAPX);
13337 match(Set dst (AndL dst src));
13338 effect(KILL cr);
13339 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13340
13341 format %{ "andq $dst, $src\t# long" %}
13342 ins_encode %{
13343 __ andq($dst$$Register, $src$$constant);
13344 %}
13345 ins_pipe(ialu_reg);
13346 %}
13347
13348 instruct andL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
13349 %{
13350 predicate(UseAPX);
13351 match(Set dst (AndL src1 src2));
13352 effect(KILL cr);
13353 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13354
13355 format %{ "eandq $dst, $src1, $src2\t# long ndd" %}
13356 ins_encode %{
13357 __ eandq($dst$$Register, $src1$$Register, $src2$$constant, false);
13358 %}
13359 ins_pipe(ialu_reg);
13360 %}
13361
13362 // And Register with Memory
13363 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
13364 %{
13365 match(Set dst (AndL dst (LoadL src)));
13366 effect(KILL cr);
13367 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13368
13369 ins_cost(150);
13370 format %{ "andq $dst, $src\t# long" %}
13371 ins_encode %{
13372 __ andq($dst$$Register, $src$$Address);
13373 %}
13374 ins_pipe(ialu_reg_mem);
13375 %}
13376
13377 // And Memory with Register
13378 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
13379 %{
13380 match(Set dst (StoreL dst (AndL (LoadL dst) src)));
13381 effect(KILL cr);
13382 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13383
13384 ins_cost(150);
13385 format %{ "andq $dst, $src\t# long" %}
13386 ins_encode %{
13387 __ andq($dst$$Address, $src$$Register);
13388 %}
13389 ins_pipe(ialu_mem_reg);
13390 %}
13391
13392 // And Memory with Immediate
13393 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
13394 %{
13395 match(Set dst (StoreL dst (AndL (LoadL dst) src)));
13396 effect(KILL cr);
13397 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13398
13399 ins_cost(125);
13400 format %{ "andq $dst, $src\t# long" %}
13401 ins_encode %{
13402 __ andq($dst$$Address, $src$$constant);
13403 %}
13404 ins_pipe(ialu_mem_imm);
13405 %}
13406
13407 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
13408 %{
13409 // con should be a pure 64-bit immediate given that not(con) is a power of 2
13410 // because AND/OR works well enough for 8/32-bit values.
13411 predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
13412
13413 match(Set dst (StoreL dst (AndL (LoadL dst) con)));
13414 effect(KILL cr);
13415
13416 ins_cost(125);
13417 format %{ "btrq $dst, log2(not($con))\t# long" %}
13418 ins_encode %{
13419 __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
13420 %}
13421 ins_pipe(ialu_mem_imm);
13422 %}
13423
13424 // BMI1 instructions
13425 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
13426 match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
13427 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13428 effect(KILL cr);
13429 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13430
13431 ins_cost(125);
13432 format %{ "andnq $dst, $src1, $src2" %}
13433
13434 ins_encode %{
13435 __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
13436 %}
13437 ins_pipe(ialu_reg_mem);
13438 %}
13439
13440 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
13441 match(Set dst (AndL (XorL src1 minus_1) src2));
13442 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13443 effect(KILL cr);
13444 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13445
13446 format %{ "andnq $dst, $src1, $src2" %}
13447
13448 ins_encode %{
13449 __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
13450 %}
13451 ins_pipe(ialu_reg_mem);
13452 %}
13453
13454 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
13455 match(Set dst (AndL (SubL imm_zero src) src));
13456 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13457 effect(KILL cr);
13458 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13459
13460 format %{ "blsiq $dst, $src" %}
13461
13462 ins_encode %{
13463 __ blsiq($dst$$Register, $src$$Register);
13464 %}
13465 ins_pipe(ialu_reg);
13466 %}
13467
13468 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
13469 match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
13470 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13471 effect(KILL cr);
13472 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13473
13474 ins_cost(125);
13475 format %{ "blsiq $dst, $src" %}
13476
13477 ins_encode %{
13478 __ blsiq($dst$$Register, $src$$Address);
13479 %}
13480 ins_pipe(ialu_reg_mem);
13481 %}
13482
13483 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
13484 %{
13485 match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
13486 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13487 effect(KILL cr);
13488 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
13489
13490 ins_cost(125);
13491 format %{ "blsmskq $dst, $src" %}
13492
13493 ins_encode %{
13494 __ blsmskq($dst$$Register, $src$$Address);
13495 %}
13496 ins_pipe(ialu_reg_mem);
13497 %}
13498
13499 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
13500 %{
13501 match(Set dst (XorL (AddL src minus_1) src));
13502 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13503 effect(KILL cr);
13504 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
13505
13506 format %{ "blsmskq $dst, $src" %}
13507
13508 ins_encode %{
13509 __ blsmskq($dst$$Register, $src$$Register);
13510 %}
13511
13512 ins_pipe(ialu_reg);
13513 %}
13514
13515 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
13516 %{
13517 match(Set dst (AndL (AddL src minus_1) src) );
13518 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13519 effect(KILL cr);
13520 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13521
13522 format %{ "blsrq $dst, $src" %}
13523
13524 ins_encode %{
13525 __ blsrq($dst$$Register, $src$$Register);
13526 %}
13527
13528 ins_pipe(ialu_reg);
13529 %}
13530
13531 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
13532 %{
13533 match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
13534 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13535 effect(KILL cr);
13536 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13537
13538 ins_cost(125);
13539 format %{ "blsrq $dst, $src" %}
13540
13541 ins_encode %{
13542 __ blsrq($dst$$Register, $src$$Address);
13543 %}
13544
13545 ins_pipe(ialu_reg);
13546 %}
13547
13548 // Or Instructions
13549 // Or Register with Register
13550 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
13551 %{
13552 predicate(!UseAPX);
13553 match(Set dst (OrL dst src));
13554 effect(KILL cr);
13555 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13556
13557 format %{ "orq $dst, $src\t# long" %}
13558 ins_encode %{
13559 __ orq($dst$$Register, $src$$Register);
13560 %}
13561 ins_pipe(ialu_reg_reg);
13562 %}
13563
13564 // Or Register with Register using New Data Destination (NDD)
13565 instruct orL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
13566 %{
13567 predicate(UseAPX);
13568 match(Set dst (OrL src1 src2));
13569 effect(KILL cr);
13570 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13571
13572 format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
13573 ins_encode %{
13574 __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
13575
13576 %}
13577 ins_pipe(ialu_reg_reg);
13578 %}
13579
13580 // Use any_RegP to match R15 (TLS register) without spilling.
13581 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
13582 predicate(!UseAPX);
13583 match(Set dst (OrL dst (CastP2X src)));
13584 effect(KILL cr);
13585 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13586
13587 format %{ "orq $dst, $src\t# long" %}
13588 ins_encode %{
13589 __ orq($dst$$Register, $src$$Register);
13590 %}
13591 ins_pipe(ialu_reg_reg);
13592 %}
13593
13594 instruct orL_rReg_castP2X_ndd(rRegL dst, any_RegP src1, any_RegP src2, rFlagsReg cr) %{
13595 predicate(UseAPX);
13596 match(Set dst (OrL src1 (CastP2X src2)));
13597 effect(KILL cr);
13598 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13599
13600 format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
13601 ins_encode %{
13602 __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
13603 %}
13604 ins_pipe(ialu_reg_reg);
13605 %}
13606
13607 // Or Register with Immediate
13608 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
13609 %{
13610 predicate(!UseAPX);
13611 match(Set dst (OrL dst src));
13612 effect(KILL cr);
13613 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13614
13615 format %{ "orq $dst, $src\t# long" %}
13616 ins_encode %{
13617 __ orq($dst$$Register, $src$$constant);
13618 %}
13619 ins_pipe(ialu_reg);
13620 %}
13621
13622 instruct orL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
13623 %{
13624 predicate(UseAPX);
13625 match(Set dst (OrL src1 src2));
13626 effect(KILL cr);
13627 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13628
13629 format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
13630 ins_encode %{
13631 __ eorq($dst$$Register, $src1$$Register, $src2$$constant, false);
13632 %}
13633 ins_pipe(ialu_reg);
13634 %}
13635
13636 instruct orL_rReg_imm_rReg_ndd(rRegL dst, immL32 src1, rRegL src2, rFlagsReg cr)
13637 %{
13638 predicate(UseAPX);
13639 match(Set dst (OrL src1 src2));
13640 effect(KILL cr);
13641 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13642
13643 format %{ "eorq $dst, $src2, $src1\t# long ndd" %}
13644 ins_encode %{
13645 __ eorq($dst$$Register, $src2$$Register, $src1$$constant, false);
13646 %}
13647 ins_pipe(ialu_reg);
13648 %}
13649
13650 // Or Register with Memory
13651 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
13652 %{
13653 match(Set dst (OrL dst (LoadL src)));
13654 effect(KILL cr);
13655 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13656
13657 ins_cost(150);
13658 format %{ "orq $dst, $src\t# long" %}
13659 ins_encode %{
13660 __ orq($dst$$Register, $src$$Address);
13661 %}
13662 ins_pipe(ialu_reg_mem);
13663 %}
13664
13665 // Or Memory with Register
13666 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
13667 %{
13668 match(Set dst (StoreL dst (OrL (LoadL dst) src)));
13669 effect(KILL cr);
13670 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13671
13672 ins_cost(150);
13673 format %{ "orq $dst, $src\t# long" %}
13674 ins_encode %{
13675 __ orq($dst$$Address, $src$$Register);
13676 %}
13677 ins_pipe(ialu_mem_reg);
13678 %}
13679
13680 // Or Memory with Immediate
13681 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
13682 %{
13683 match(Set dst (StoreL dst (OrL (LoadL dst) src)));
13684 effect(KILL cr);
13685 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13686
13687 ins_cost(125);
13688 format %{ "orq $dst, $src\t# long" %}
13689 ins_encode %{
13690 __ orq($dst$$Address, $src$$constant);
13691 %}
13692 ins_pipe(ialu_mem_imm);
13693 %}
13694
13695 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
13696 %{
13697 // con should be a pure 64-bit power of 2 immediate
13698 // because AND/OR works well enough for 8/32-bit values.
13699 predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
13700
13701 match(Set dst (StoreL dst (OrL (LoadL dst) con)));
13702 effect(KILL cr);
13703
13704 ins_cost(125);
13705 format %{ "btsq $dst, log2($con)\t# long" %}
13706 ins_encode %{
13707 __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
13708 %}
13709 ins_pipe(ialu_mem_imm);
13710 %}
13711
13712 // Xor Instructions
13713 // Xor Register with Register
13714 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
13715 %{
13716 predicate(!UseAPX);
13717 match(Set dst (XorL dst src));
13718 effect(KILL cr);
13719 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13720
13721 format %{ "xorq $dst, $src\t# long" %}
13722 ins_encode %{
13723 __ xorq($dst$$Register, $src$$Register);
13724 %}
13725 ins_pipe(ialu_reg_reg);
13726 %}
13727
13728 // Xor Register with Register using New Data Destination (NDD)
13729 instruct xorL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
13730 %{
13731 predicate(UseAPX);
13732 match(Set dst (XorL src1 src2));
13733 effect(KILL cr);
13734 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13735
13736 format %{ "exorq $dst, $src1, $src2\t# long ndd" %}
13737 ins_encode %{
13738 __ exorq($dst$$Register, $src1$$Register, $src2$$Register, false);
13739 %}
13740 ins_pipe(ialu_reg_reg);
13741 %}
13742
13743 // Xor Register with Immediate -1
13744 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm)
13745 %{
13746 predicate(!UseAPX);
13747 match(Set dst (XorL dst imm));
13748
13749 format %{ "notq $dst" %}
13750 ins_encode %{
13751 __ notq($dst$$Register);
13752 %}
13753 ins_pipe(ialu_reg);
13754 %}
13755
13756 instruct xorL_rReg_im1_ndd(rRegL dst,rRegL src, immL_M1 imm)
13757 %{
13758 predicate(UseAPX);
13759 match(Set dst (XorL src imm));
13760 flag(PD::Flag_ndd_demotable_opr1);
13761
13762 format %{ "enotq $dst, $src" %}
13763 ins_encode %{
13764 __ enotq($dst$$Register, $src$$Register);
13765 %}
13766 ins_pipe(ialu_reg);
13767 %}
13768
13769 // Xor Register with Immediate
13770 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
13771 %{
13772 // Strict predicate check to make selection of xorL_rReg_im1 cost agnostic if immL32 src is -1.
13773 predicate(!UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
13774 match(Set dst (XorL dst src));
13775 effect(KILL cr);
13776 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13777
13778 format %{ "xorq $dst, $src\t# long" %}
13779 ins_encode %{
13780 __ xorq($dst$$Register, $src$$constant);
13781 %}
13782 ins_pipe(ialu_reg);
13783 %}
13784
13785 instruct xorL_rReg_rReg_imm(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
13786 %{
13787 // Strict predicate check to make selection of xorL_rReg_im1_ndd cost agnostic if immL32 src2 is -1.
13788 predicate(UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
13789 match(Set dst (XorL src1 src2));
13790 effect(KILL cr);
13791 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13792
13793 format %{ "exorq $dst, $src1, $src2\t# long ndd" %}
13794 ins_encode %{
13795 __ exorq($dst$$Register, $src1$$Register, $src2$$constant, false);
13796 %}
13797 ins_pipe(ialu_reg);
13798 %}
13799
13800 // Xor Register with Memory
13801 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
13802 %{
13803 match(Set dst (XorL dst (LoadL src)));
13804 effect(KILL cr);
13805 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13806
13807 ins_cost(150);
13808 format %{ "xorq $dst, $src\t# long" %}
13809 ins_encode %{
13810 __ xorq($dst$$Register, $src$$Address);
13811 %}
13812 ins_pipe(ialu_reg_mem);
13813 %}
13814
13815 // Xor Memory with Register
13816 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
13817 %{
13818 match(Set dst (StoreL dst (XorL (LoadL dst) src)));
13819 effect(KILL cr);
13820 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13821
13822 ins_cost(150);
13823 format %{ "xorq $dst, $src\t# long" %}
13824 ins_encode %{
13825 __ xorq($dst$$Address, $src$$Register);
13826 %}
13827 ins_pipe(ialu_mem_reg);
13828 %}
13829
13830 // Xor Memory with Immediate
13831 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
13832 %{
13833 match(Set dst (StoreL dst (XorL (LoadL dst) src)));
13834 effect(KILL cr);
13835 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13836
13837 ins_cost(125);
13838 format %{ "xorq $dst, $src\t# long" %}
13839 ins_encode %{
13840 __ xorq($dst$$Address, $src$$constant);
13841 %}
13842 ins_pipe(ialu_mem_imm);
13843 %}
13844
13845 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
13846 %{
13847 match(Set dst (CmpLTMask p q));
13848 effect(KILL cr);
13849
13850 ins_cost(400);
13851 format %{ "cmpl $p, $q\t# cmpLTMask\n\t"
13852 "setcc $dst \t# emits setlt + movzbl or setzul for APX"
13853 "negl $dst" %}
13854 ins_encode %{
13855 __ cmpl($p$$Register, $q$$Register);
13856 __ setcc(Assembler::less, $dst$$Register);
13857 __ negl($dst$$Register);
13858 %}
13859 ins_pipe(pipe_slow);
13860 %}
13861
13862 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
13863 %{
13864 match(Set dst (CmpLTMask dst zero));
13865 effect(KILL cr);
13866
13867 ins_cost(100);
13868 format %{ "sarl $dst, #31\t# cmpLTMask0" %}
13869 ins_encode %{
13870 __ sarl($dst$$Register, 31);
13871 %}
13872 ins_pipe(ialu_reg);
13873 %}
13874
13875 /* Better to save a register than avoid a branch */
13876 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
13877 %{
13878 match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
13879 effect(KILL cr);
13880 ins_cost(300);
13881 format %{ "subl $p,$q\t# cadd_cmpLTMask\n\t"
13882 "jge done\n\t"
13883 "addl $p,$y\n"
13884 "done: " %}
13885 ins_encode %{
13886 Register Rp = $p$$Register;
13887 Register Rq = $q$$Register;
13888 Register Ry = $y$$Register;
13889 Label done;
13890 __ subl(Rp, Rq);
13891 __ jccb(Assembler::greaterEqual, done);
13892 __ addl(Rp, Ry);
13893 __ bind(done);
13894 %}
13895 ins_pipe(pipe_cmplt);
13896 %}
13897
13898 /* Better to save a register than avoid a branch */
13899 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
13900 %{
13901 match(Set y (AndI (CmpLTMask p q) y));
13902 effect(KILL cr);
13903
13904 ins_cost(300);
13905
13906 format %{ "cmpl $p, $q\t# and_cmpLTMask\n\t"
13907 "jlt done\n\t"
13908 "xorl $y, $y\n"
13909 "done: " %}
13910 ins_encode %{
13911 Register Rp = $p$$Register;
13912 Register Rq = $q$$Register;
13913 Register Ry = $y$$Register;
13914 Label done;
13915 __ cmpl(Rp, Rq);
13916 __ jccb(Assembler::less, done);
13917 __ xorl(Ry, Ry);
13918 __ bind(done);
13919 %}
13920 ins_pipe(pipe_cmplt);
13921 %}
13922
13923
13924 //---------- FP Instructions------------------------------------------------
13925
13926 // Really expensive, avoid
13927 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
13928 %{
13929 match(Set cr (CmpF src1 src2));
13930
13931 ins_cost(500);
13932 format %{ "ucomiss $src1, $src2\n\t"
13933 "jnp,s exit\n\t"
13934 "pushfq\t# saw NaN, set CF\n\t"
13935 "andq [rsp], #0xffffff2b\n\t"
13936 "popfq\n"
13937 "exit:" %}
13938 ins_encode %{
13939 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
13940 emit_cmpfp_fixup(masm);
13941 %}
13942 ins_pipe(pipe_slow);
13943 %}
13944
13945 instruct cmpF_cc_regCF(rFlagsRegUCF cr, regF src1, regF src2) %{
13946 match(Set cr (CmpF src1 src2));
13947
13948 ins_cost(100);
13949 format %{ "ucomiss $src1, $src2" %}
13950 ins_encode %{
13951 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
13952 %}
13953 ins_pipe(pipe_slow);
13954 %}
13955
13956 instruct cmpF_cc_regCFE(rFlagsRegUCFE cr, regF src1, regF src2) %{
13957 match(Set cr (CmpF src1 src2));
13958
13959 ins_cost(100);
13960 format %{ "evucomxss $src1, $src2" %}
13961 ins_encode %{
13962 __ evucomxss($src1$$XMMRegister, $src2$$XMMRegister);
13963 %}
13964 ins_pipe(pipe_slow);
13965 %}
13966
13967 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
13968 match(Set cr (CmpF src1 (LoadF src2)));
13969
13970 ins_cost(100);
13971 format %{ "ucomiss $src1, $src2" %}
13972 ins_encode %{
13973 __ ucomiss($src1$$XMMRegister, $src2$$Address);
13974 %}
13975 ins_pipe(pipe_slow);
13976 %}
13977
13978 instruct cmpF_cc_memCFE(rFlagsRegUCFE cr, regF src1, memory src2) %{
13979 match(Set cr (CmpF src1 (LoadF src2)));
13980
13981 ins_cost(100);
13982 format %{ "evucomxss $src1, $src2" %}
13983 ins_encode %{
13984 __ evucomxss($src1$$XMMRegister, $src2$$Address);
13985 %}
13986 ins_pipe(pipe_slow);
13987 %}
13988
13989 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
13990 match(Set cr (CmpF src con));
13991
13992 ins_cost(100);
13993 format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
13994 ins_encode %{
13995 __ ucomiss($src$$XMMRegister, $constantaddress($con));
13996 %}
13997 ins_pipe(pipe_slow);
13998 %}
13999
14000 instruct cmpF_cc_immCFE(rFlagsRegUCFE cr, regF src, immF con) %{
14001 match(Set cr (CmpF src con));
14002
14003 ins_cost(100);
14004 format %{ "evucomxss $src, [$constantaddress]\t# load from constant table: float=$con" %}
14005 ins_encode %{
14006 __ evucomxss($src$$XMMRegister, $constantaddress($con));
14007 %}
14008 ins_pipe(pipe_slow);
14009 %}
14010
14011 // Really expensive, avoid
14012 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
14013 %{
14014 match(Set cr (CmpD src1 src2));
14015
14016 ins_cost(500);
14017 format %{ "ucomisd $src1, $src2\n\t"
14018 "jnp,s exit\n\t"
14019 "pushfq\t# saw NaN, set CF\n\t"
14020 "andq [rsp], #0xffffff2b\n\t"
14021 "popfq\n"
14022 "exit:" %}
14023 ins_encode %{
14024 __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
14025 emit_cmpfp_fixup(masm);
14026 %}
14027 ins_pipe(pipe_slow);
14028 %}
14029
14030 instruct cmpD_cc_regCF(rFlagsRegUCF cr, regD src1, regD src2) %{
14031 match(Set cr (CmpD src1 src2));
14032
14033 ins_cost(100);
14034 format %{ "ucomisd $src1, $src2 test" %}
14035 ins_encode %{
14036 __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
14037 %}
14038 ins_pipe(pipe_slow);
14039 %}
14040
14041 instruct cmpD_cc_regCFE(rFlagsRegUCFE cr, regD src1, regD src2) %{
14042 match(Set cr (CmpD src1 src2));
14043
14044 ins_cost(100);
14045 format %{ "evucomxsd $src1, $src2 test" %}
14046 ins_encode %{
14047 __ evucomxsd($src1$$XMMRegister, $src2$$XMMRegister);
14048 %}
14049 ins_pipe(pipe_slow);
14050 %}
14051
14052 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
14053 match(Set cr (CmpD src1 (LoadD src2)));
14054
14055 ins_cost(100);
14056 format %{ "ucomisd $src1, $src2" %}
14057 ins_encode %{
14058 __ ucomisd($src1$$XMMRegister, $src2$$Address);
14059 %}
14060 ins_pipe(pipe_slow);
14061 %}
14062
14063 instruct cmpD_cc_memCFE(rFlagsRegUCFE cr, regD src1, memory src2) %{
14064 match(Set cr (CmpD src1 (LoadD src2)));
14065
14066 ins_cost(100);
14067 format %{ "evucomxsd $src1, $src2" %}
14068 ins_encode %{
14069 __ evucomxsd($src1$$XMMRegister, $src2$$Address);
14070 %}
14071 ins_pipe(pipe_slow);
14072 %}
14073
14074 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
14075 match(Set cr (CmpD src con));
14076 ins_cost(100);
14077 format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
14078 ins_encode %{
14079 __ ucomisd($src$$XMMRegister, $constantaddress($con));
14080 %}
14081 ins_pipe(pipe_slow);
14082 %}
14083
14084 instruct cmpD_cc_immCFE(rFlagsRegUCFE cr, regD src, immD con) %{
14085 match(Set cr (CmpD src con));
14086
14087 ins_cost(100);
14088 format %{ "evucomxsd $src, [$constantaddress]\t# load from constant table: double=$con" %}
14089 ins_encode %{
14090 __ evucomxsd($src$$XMMRegister, $constantaddress($con));
14091 %}
14092 ins_pipe(pipe_slow);
14093 %}
14094
14095 // Compare into -1,0,1
14096 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
14097 %{
14098 match(Set dst (CmpF3 src1 src2));
14099 effect(KILL cr);
14100
14101 ins_cost(275);
14102 format %{ "ucomiss $src1, $src2\n\t"
14103 "movl $dst, #-1\n\t"
14104 "jp,s done\n\t"
14105 "jb,s done\n\t"
14106 "setne $dst\n\t"
14107 "movzbl $dst, $dst\n"
14108 "done:" %}
14109 ins_encode %{
14110 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
14111 emit_cmpfp3(masm, $dst$$Register);
14112 %}
14113 ins_pipe(pipe_slow);
14114 %}
14115
14116 // Compare into -1,0,1
14117 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
14118 %{
14119 match(Set dst (CmpF3 src1 (LoadF src2)));
14120 effect(KILL cr);
14121
14122 ins_cost(275);
14123 format %{ "ucomiss $src1, $src2\n\t"
14124 "movl $dst, #-1\n\t"
14125 "jp,s done\n\t"
14126 "jb,s done\n\t"
14127 "setne $dst\n\t"
14128 "movzbl $dst, $dst\n"
14129 "done:" %}
14130 ins_encode %{
14131 __ ucomiss($src1$$XMMRegister, $src2$$Address);
14132 emit_cmpfp3(masm, $dst$$Register);
14133 %}
14134 ins_pipe(pipe_slow);
14135 %}
14136
14137 // Compare into -1,0,1
14138 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
14139 match(Set dst (CmpF3 src con));
14140 effect(KILL cr);
14141
14142 ins_cost(275);
14143 format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
14144 "movl $dst, #-1\n\t"
14145 "jp,s done\n\t"
14146 "jb,s done\n\t"
14147 "setne $dst\n\t"
14148 "movzbl $dst, $dst\n"
14149 "done:" %}
14150 ins_encode %{
14151 __ ucomiss($src$$XMMRegister, $constantaddress($con));
14152 emit_cmpfp3(masm, $dst$$Register);
14153 %}
14154 ins_pipe(pipe_slow);
14155 %}
14156
14157 // Compare into -1,0,1
14158 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
14159 %{
14160 match(Set dst (CmpD3 src1 src2));
14161 effect(KILL cr);
14162
14163 ins_cost(275);
14164 format %{ "ucomisd $src1, $src2\n\t"
14165 "movl $dst, #-1\n\t"
14166 "jp,s done\n\t"
14167 "jb,s done\n\t"
14168 "setne $dst\n\t"
14169 "movzbl $dst, $dst\n"
14170 "done:" %}
14171 ins_encode %{
14172 __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
14173 emit_cmpfp3(masm, $dst$$Register);
14174 %}
14175 ins_pipe(pipe_slow);
14176 %}
14177
14178 // Compare into -1,0,1
14179 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
14180 %{
14181 match(Set dst (CmpD3 src1 (LoadD src2)));
14182 effect(KILL cr);
14183
14184 ins_cost(275);
14185 format %{ "ucomisd $src1, $src2\n\t"
14186 "movl $dst, #-1\n\t"
14187 "jp,s done\n\t"
14188 "jb,s done\n\t"
14189 "setne $dst\n\t"
14190 "movzbl $dst, $dst\n"
14191 "done:" %}
14192 ins_encode %{
14193 __ ucomisd($src1$$XMMRegister, $src2$$Address);
14194 emit_cmpfp3(masm, $dst$$Register);
14195 %}
14196 ins_pipe(pipe_slow);
14197 %}
14198
14199 // Compare into -1,0,1
14200 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
14201 match(Set dst (CmpD3 src con));
14202 effect(KILL cr);
14203
14204 ins_cost(275);
14205 format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
14206 "movl $dst, #-1\n\t"
14207 "jp,s done\n\t"
14208 "jb,s done\n\t"
14209 "setne $dst\n\t"
14210 "movzbl $dst, $dst\n"
14211 "done:" %}
14212 ins_encode %{
14213 __ ucomisd($src$$XMMRegister, $constantaddress($con));
14214 emit_cmpfp3(masm, $dst$$Register);
14215 %}
14216 ins_pipe(pipe_slow);
14217 %}
14218
14219 //----------Arithmetic Conversion Instructions---------------------------------
14220
14221 instruct convF2D_reg_reg(regD dst, regF src)
14222 %{
14223 match(Set dst (ConvF2D src));
14224
14225 format %{ "cvtss2sd $dst, $src" %}
14226 ins_encode %{
14227 __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
14228 %}
14229 ins_pipe(pipe_slow); // XXX
14230 %}
14231
14232 instruct convF2D_reg_mem(regD dst, memory src)
14233 %{
14234 predicate(UseAVX == 0);
14235 match(Set dst (ConvF2D (LoadF src)));
14236
14237 format %{ "cvtss2sd $dst, $src" %}
14238 ins_encode %{
14239 __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
14240 %}
14241 ins_pipe(pipe_slow); // XXX
14242 %}
14243
14244 instruct convD2F_reg_reg(regF dst, regD src)
14245 %{
14246 match(Set dst (ConvD2F src));
14247
14248 format %{ "cvtsd2ss $dst, $src" %}
14249 ins_encode %{
14250 __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
14251 %}
14252 ins_pipe(pipe_slow); // XXX
14253 %}
14254
14255 instruct convD2F_reg_mem(regF dst, memory src)
14256 %{
14257 predicate(UseAVX == 0);
14258 match(Set dst (ConvD2F (LoadD src)));
14259
14260 format %{ "cvtsd2ss $dst, $src" %}
14261 ins_encode %{
14262 __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
14263 %}
14264 ins_pipe(pipe_slow); // XXX
14265 %}
14266
14267 // XXX do mem variants
14268 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
14269 %{
14270 predicate(!VM_Version::supports_avx10_2());
14271 match(Set dst (ConvF2I src));
14272 effect(KILL cr);
14273 format %{ "convert_f2i $dst, $src" %}
14274 ins_encode %{
14275 __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
14276 %}
14277 ins_pipe(pipe_slow);
14278 %}
14279
14280 instruct convF2I_reg_reg_avx10_2(rRegI dst, regF src)
14281 %{
14282 predicate(VM_Version::supports_avx10_2());
14283 match(Set dst (ConvF2I src));
14284 format %{ "evcvttss2sisl $dst, $src" %}
14285 ins_encode %{
14286 __ evcvttss2sisl($dst$$Register, $src$$XMMRegister);
14287 %}
14288 ins_pipe(pipe_slow);
14289 %}
14290
14291 instruct convF2I_reg_mem_avx10_2(rRegI dst, memory src)
14292 %{
14293 predicate(VM_Version::supports_avx10_2());
14294 match(Set dst (ConvF2I (LoadF src)));
14295 format %{ "evcvttss2sisl $dst, $src" %}
14296 ins_encode %{
14297 __ evcvttss2sisl($dst$$Register, $src$$Address);
14298 %}
14299 ins_pipe(pipe_slow);
14300 %}
14301
14302 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
14303 %{
14304 predicate(!VM_Version::supports_avx10_2());
14305 match(Set dst (ConvF2L src));
14306 effect(KILL cr);
14307 format %{ "convert_f2l $dst, $src"%}
14308 ins_encode %{
14309 __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
14310 %}
14311 ins_pipe(pipe_slow);
14312 %}
14313
14314 instruct convF2L_reg_reg_avx10_2(rRegL dst, regF src)
14315 %{
14316 predicate(VM_Version::supports_avx10_2());
14317 match(Set dst (ConvF2L src));
14318 format %{ "evcvttss2sisq $dst, $src" %}
14319 ins_encode %{
14320 __ evcvttss2sisq($dst$$Register, $src$$XMMRegister);
14321 %}
14322 ins_pipe(pipe_slow);
14323 %}
14324
14325 instruct convF2L_reg_mem_avx10_2(rRegL dst, memory src)
14326 %{
14327 predicate(VM_Version::supports_avx10_2());
14328 match(Set dst (ConvF2L (LoadF src)));
14329 format %{ "evcvttss2sisq $dst, $src" %}
14330 ins_encode %{
14331 __ evcvttss2sisq($dst$$Register, $src$$Address);
14332 %}
14333 ins_pipe(pipe_slow);
14334 %}
14335
14336 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
14337 %{
14338 predicate(!VM_Version::supports_avx10_2());
14339 match(Set dst (ConvD2I src));
14340 effect(KILL cr);
14341 format %{ "convert_d2i $dst, $src"%}
14342 ins_encode %{
14343 __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
14344 %}
14345 ins_pipe(pipe_slow);
14346 %}
14347
14348 instruct convD2I_reg_reg_avx10_2(rRegI dst, regD src)
14349 %{
14350 predicate(VM_Version::supports_avx10_2());
14351 match(Set dst (ConvD2I src));
14352 format %{ "evcvttsd2sisl $dst, $src" %}
14353 ins_encode %{
14354 __ evcvttsd2sisl($dst$$Register, $src$$XMMRegister);
14355 %}
14356 ins_pipe(pipe_slow);
14357 %}
14358
14359 instruct convD2I_reg_mem_avx10_2(rRegI dst, memory src)
14360 %{
14361 predicate(VM_Version::supports_avx10_2());
14362 match(Set dst (ConvD2I (LoadD src)));
14363 format %{ "evcvttsd2sisl $dst, $src" %}
14364 ins_encode %{
14365 __ evcvttsd2sisl($dst$$Register, $src$$Address);
14366 %}
14367 ins_pipe(pipe_slow);
14368 %}
14369
14370 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
14371 %{
14372 predicate(!VM_Version::supports_avx10_2());
14373 match(Set dst (ConvD2L src));
14374 effect(KILL cr);
14375 format %{ "convert_d2l $dst, $src"%}
14376 ins_encode %{
14377 __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
14378 %}
14379 ins_pipe(pipe_slow);
14380 %}
14381
14382 instruct convD2L_reg_reg_avx10_2(rRegL dst, regD src)
14383 %{
14384 predicate(VM_Version::supports_avx10_2());
14385 match(Set dst (ConvD2L src));
14386 format %{ "evcvttsd2sisq $dst, $src" %}
14387 ins_encode %{
14388 __ evcvttsd2sisq($dst$$Register, $src$$XMMRegister);
14389 %}
14390 ins_pipe(pipe_slow);
14391 %}
14392
14393 instruct convD2L_reg_mem_avx10_2(rRegL dst, memory src)
14394 %{
14395 predicate(VM_Version::supports_avx10_2());
14396 match(Set dst (ConvD2L (LoadD src)));
14397 format %{ "evcvttsd2sisq $dst, $src" %}
14398 ins_encode %{
14399 __ evcvttsd2sisq($dst$$Register, $src$$Address);
14400 %}
14401 ins_pipe(pipe_slow);
14402 %}
14403
14404 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
14405 %{
14406 match(Set dst (RoundD src));
14407 effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
14408 format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
14409 ins_encode %{
14410 __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
14411 %}
14412 ins_pipe(pipe_slow);
14413 %}
14414
14415 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
14416 %{
14417 match(Set dst (RoundF src));
14418 effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
14419 format %{ "round_float $dst,$src" %}
14420 ins_encode %{
14421 __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
14422 %}
14423 ins_pipe(pipe_slow);
14424 %}
14425
14426 instruct convI2F_reg_reg(vlRegF dst, rRegI src)
14427 %{
14428 predicate(!UseXmmI2F);
14429 match(Set dst (ConvI2F src));
14430
14431 format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
14432 ins_encode %{
14433 if (UseAVX > 0) {
14434 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14435 }
14436 __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
14437 %}
14438 ins_pipe(pipe_slow); // XXX
14439 %}
14440
14441 instruct convI2F_reg_mem(regF dst, memory src)
14442 %{
14443 predicate(UseAVX == 0);
14444 match(Set dst (ConvI2F (LoadI src)));
14445
14446 format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
14447 ins_encode %{
14448 __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
14449 %}
14450 ins_pipe(pipe_slow); // XXX
14451 %}
14452
14453 instruct convI2D_reg_reg(vlRegD dst, rRegI src)
14454 %{
14455 predicate(!UseXmmI2D);
14456 match(Set dst (ConvI2D src));
14457
14458 format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
14459 ins_encode %{
14460 if (UseAVX > 0) {
14461 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14462 }
14463 __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
14464 %}
14465 ins_pipe(pipe_slow); // XXX
14466 %}
14467
14468 instruct convI2D_reg_mem(regD dst, memory src)
14469 %{
14470 predicate(UseAVX == 0);
14471 match(Set dst (ConvI2D (LoadI src)));
14472
14473 format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
14474 ins_encode %{
14475 __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
14476 %}
14477 ins_pipe(pipe_slow); // XXX
14478 %}
14479
14480 instruct convXI2F_reg(regF dst, rRegI src)
14481 %{
14482 predicate(UseXmmI2F);
14483 match(Set dst (ConvI2F src));
14484
14485 format %{ "movdl $dst, $src\n\t"
14486 "cvtdq2psl $dst, $dst\t# i2f" %}
14487 ins_encode %{
14488 __ movdl($dst$$XMMRegister, $src$$Register);
14489 __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
14490 %}
14491 ins_pipe(pipe_slow); // XXX
14492 %}
14493
14494 instruct convXI2D_reg(regD dst, rRegI src)
14495 %{
14496 predicate(UseXmmI2D);
14497 match(Set dst (ConvI2D src));
14498
14499 format %{ "movdl $dst, $src\n\t"
14500 "cvtdq2pdl $dst, $dst\t# i2d" %}
14501 ins_encode %{
14502 __ movdl($dst$$XMMRegister, $src$$Register);
14503 __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
14504 %}
14505 ins_pipe(pipe_slow); // XXX
14506 %}
14507
14508 instruct convL2F_reg_reg(vlRegF dst, rRegL src)
14509 %{
14510 match(Set dst (ConvL2F src));
14511
14512 format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
14513 ins_encode %{
14514 if (UseAVX > 0) {
14515 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14516 }
14517 __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
14518 %}
14519 ins_pipe(pipe_slow); // XXX
14520 %}
14521
14522 instruct convL2F_reg_mem(regF dst, memory src)
14523 %{
14524 predicate(UseAVX == 0);
14525 match(Set dst (ConvL2F (LoadL src)));
14526
14527 format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
14528 ins_encode %{
14529 __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
14530 %}
14531 ins_pipe(pipe_slow); // XXX
14532 %}
14533
14534 instruct convL2D_reg_reg(vlRegD dst, rRegL src)
14535 %{
14536 match(Set dst (ConvL2D src));
14537
14538 format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
14539 ins_encode %{
14540 if (UseAVX > 0) {
14541 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14542 }
14543 __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
14544 %}
14545 ins_pipe(pipe_slow); // XXX
14546 %}
14547
14548 instruct convL2D_reg_mem(regD dst, memory src)
14549 %{
14550 predicate(UseAVX == 0);
14551 match(Set dst (ConvL2D (LoadL src)));
14552
14553 format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
14554 ins_encode %{
14555 __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
14556 %}
14557 ins_pipe(pipe_slow); // XXX
14558 %}
14559
14560 instruct convI2L_reg_reg(rRegL dst, rRegI src)
14561 %{
14562 match(Set dst (ConvI2L src));
14563
14564 ins_cost(125);
14565 format %{ "movslq $dst, $src\t# i2l" %}
14566 ins_encode %{
14567 __ movslq($dst$$Register, $src$$Register);
14568 %}
14569 ins_pipe(ialu_reg_reg);
14570 %}
14571
14572 // Zero-extend convert int to long
14573 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
14574 %{
14575 match(Set dst (AndL (ConvI2L src) mask));
14576
14577 format %{ "movl $dst, $src\t# i2l zero-extend\n\t" %}
14578 ins_encode %{
14579 if ($dst$$reg != $src$$reg) {
14580 __ movl($dst$$Register, $src$$Register);
14581 }
14582 %}
14583 ins_pipe(ialu_reg_reg);
14584 %}
14585
14586 // Zero-extend convert int to long
14587 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
14588 %{
14589 match(Set dst (AndL (ConvI2L (LoadI src)) mask));
14590
14591 format %{ "movl $dst, $src\t# i2l zero-extend\n\t" %}
14592 ins_encode %{
14593 __ movl($dst$$Register, $src$$Address);
14594 %}
14595 ins_pipe(ialu_reg_mem);
14596 %}
14597
14598 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
14599 %{
14600 match(Set dst (AndL src mask));
14601
14602 format %{ "movl $dst, $src\t# zero-extend long" %}
14603 ins_encode %{
14604 __ movl($dst$$Register, $src$$Register);
14605 %}
14606 ins_pipe(ialu_reg_reg);
14607 %}
14608
14609 instruct convL2I_reg_reg(rRegI dst, rRegL src)
14610 %{
14611 match(Set dst (ConvL2I src));
14612
14613 format %{ "movl $dst, $src\t# l2i" %}
14614 ins_encode %{
14615 __ movl($dst$$Register, $src$$Register);
14616 %}
14617 ins_pipe(ialu_reg_reg);
14618 %}
14619
14620
14621 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
14622 match(Set dst (MoveF2I src));
14623 effect(DEF dst, USE src);
14624
14625 ins_cost(125);
14626 format %{ "movl $dst, $src\t# MoveF2I_stack_reg" %}
14627 ins_encode %{
14628 __ movl($dst$$Register, Address(rsp, $src$$disp));
14629 %}
14630 ins_pipe(ialu_reg_mem);
14631 %}
14632
14633 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
14634 match(Set dst (MoveI2F src));
14635 effect(DEF dst, USE src);
14636
14637 ins_cost(125);
14638 format %{ "movss $dst, $src\t# MoveI2F_stack_reg" %}
14639 ins_encode %{
14640 __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
14641 %}
14642 ins_pipe(pipe_slow);
14643 %}
14644
14645 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
14646 match(Set dst (MoveD2L src));
14647 effect(DEF dst, USE src);
14648
14649 ins_cost(125);
14650 format %{ "movq $dst, $src\t# MoveD2L_stack_reg" %}
14651 ins_encode %{
14652 __ movq($dst$$Register, Address(rsp, $src$$disp));
14653 %}
14654 ins_pipe(ialu_reg_mem);
14655 %}
14656
14657 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
14658 predicate(!UseXmmLoadAndClearUpper);
14659 match(Set dst (MoveL2D src));
14660 effect(DEF dst, USE src);
14661
14662 ins_cost(125);
14663 format %{ "movlpd $dst, $src\t# MoveL2D_stack_reg" %}
14664 ins_encode %{
14665 __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
14666 %}
14667 ins_pipe(pipe_slow);
14668 %}
14669
14670 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
14671 predicate(UseXmmLoadAndClearUpper);
14672 match(Set dst (MoveL2D src));
14673 effect(DEF dst, USE src);
14674
14675 ins_cost(125);
14676 format %{ "movsd $dst, $src\t# MoveL2D_stack_reg" %}
14677 ins_encode %{
14678 __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
14679 %}
14680 ins_pipe(pipe_slow);
14681 %}
14682
14683
14684 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
14685 match(Set dst (MoveF2I src));
14686 effect(DEF dst, USE src);
14687
14688 ins_cost(95); // XXX
14689 format %{ "movss $dst, $src\t# MoveF2I_reg_stack" %}
14690 ins_encode %{
14691 __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
14692 %}
14693 ins_pipe(pipe_slow);
14694 %}
14695
14696 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
14697 match(Set dst (MoveI2F src));
14698 effect(DEF dst, USE src);
14699
14700 ins_cost(100);
14701 format %{ "movl $dst, $src\t# MoveI2F_reg_stack" %}
14702 ins_encode %{
14703 __ movl(Address(rsp, $dst$$disp), $src$$Register);
14704 %}
14705 ins_pipe( ialu_mem_reg );
14706 %}
14707
14708 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
14709 match(Set dst (MoveD2L src));
14710 effect(DEF dst, USE src);
14711
14712 ins_cost(95); // XXX
14713 format %{ "movsd $dst, $src\t# MoveL2D_reg_stack" %}
14714 ins_encode %{
14715 __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
14716 %}
14717 ins_pipe(pipe_slow);
14718 %}
14719
14720 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
14721 match(Set dst (MoveL2D src));
14722 effect(DEF dst, USE src);
14723
14724 ins_cost(100);
14725 format %{ "movq $dst, $src\t# MoveL2D_reg_stack" %}
14726 ins_encode %{
14727 __ movq(Address(rsp, $dst$$disp), $src$$Register);
14728 %}
14729 ins_pipe(ialu_mem_reg);
14730 %}
14731
14732 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
14733 match(Set dst (MoveF2I src));
14734 effect(DEF dst, USE src);
14735 ins_cost(85);
14736 format %{ "movd $dst,$src\t# MoveF2I" %}
14737 ins_encode %{
14738 __ movdl($dst$$Register, $src$$XMMRegister);
14739 %}
14740 ins_pipe( pipe_slow );
14741 %}
14742
14743 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
14744 match(Set dst (MoveD2L src));
14745 effect(DEF dst, USE src);
14746 ins_cost(85);
14747 format %{ "movd $dst,$src\t# MoveD2L" %}
14748 ins_encode %{
14749 __ movdq($dst$$Register, $src$$XMMRegister);
14750 %}
14751 ins_pipe( pipe_slow );
14752 %}
14753
14754 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
14755 match(Set dst (MoveI2F src));
14756 effect(DEF dst, USE src);
14757 ins_cost(100);
14758 format %{ "movd $dst,$src\t# MoveI2F" %}
14759 ins_encode %{
14760 __ movdl($dst$$XMMRegister, $src$$Register);
14761 %}
14762 ins_pipe( pipe_slow );
14763 %}
14764
14765 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
14766 match(Set dst (MoveL2D src));
14767 effect(DEF dst, USE src);
14768 ins_cost(100);
14769 format %{ "movd $dst,$src\t# MoveL2D" %}
14770 ins_encode %{
14771 __ movdq($dst$$XMMRegister, $src$$Register);
14772 %}
14773 ins_pipe( pipe_slow );
14774 %}
14775
14776
14777 // Fast clearing of an array
14778 // Small non-constant lenght ClearArray for non-AVX512 targets.
14779 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
14780 Universe dummy, rFlagsReg cr)
14781 %{
14782 predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
14783 match(Set dummy (ClearArray (Binary cnt base) val));
14784 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
14785
14786 format %{ $$template
14787 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14788 $$emit$$"jg LARGE\n\t"
14789 $$emit$$"dec rcx\n\t"
14790 $$emit$$"js DONE\t# Zero length\n\t"
14791 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14792 $$emit$$"dec rcx\n\t"
14793 $$emit$$"jge LOOP\n\t"
14794 $$emit$$"jmp DONE\n\t"
14795 $$emit$$"# LARGE:\n\t"
14796 if (UseFastStosb) {
14797 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14798 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14799 } else if (UseXMMForObjInit) {
14800 $$emit$$"movdq $tmp, $val\n\t"
14801 $$emit$$"punpcklqdq $tmp, $tmp\n\t"
14802 $$emit$$"vinserti128_high $tmp, $tmp\n\t"
14803 $$emit$$"jmpq L_zero_64_bytes\n\t"
14804 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14805 $$emit$$"vmovdqu $tmp,(rax)\n\t"
14806 $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
14807 $$emit$$"add 0x40,rax\n\t"
14808 $$emit$$"# L_zero_64_bytes:\n\t"
14809 $$emit$$"sub 0x8,rcx\n\t"
14810 $$emit$$"jge L_loop\n\t"
14811 $$emit$$"add 0x4,rcx\n\t"
14812 $$emit$$"jl L_tail\n\t"
14813 $$emit$$"vmovdqu $tmp,(rax)\n\t"
14814 $$emit$$"add 0x20,rax\n\t"
14815 $$emit$$"sub 0x4,rcx\n\t"
14816 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14817 $$emit$$"add 0x4,rcx\n\t"
14818 $$emit$$"jle L_end\n\t"
14819 $$emit$$"dec rcx\n\t"
14820 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14821 $$emit$$"vmovq xmm0,(rax)\n\t"
14822 $$emit$$"add 0x8,rax\n\t"
14823 $$emit$$"dec rcx\n\t"
14824 $$emit$$"jge L_sloop\n\t"
14825 $$emit$$"# L_end:\n\t"
14826 } else {
14827 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14828 }
14829 $$emit$$"# DONE"
14830 %}
14831 ins_encode %{
14832 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
14833 $tmp$$XMMRegister, false, false);
14834 %}
14835 ins_pipe(pipe_slow);
14836 %}
14837
14838 instruct rep_stos_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
14839 Universe dummy, rFlagsReg cr)
14840 %{
14841 predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
14842 match(Set dummy (ClearArray (Binary cnt base) val));
14843 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
14844
14845 format %{ $$template
14846 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14847 $$emit$$"jg LARGE\n\t"
14848 $$emit$$"dec rcx\n\t"
14849 $$emit$$"js DONE\t# Zero length\n\t"
14850 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14851 $$emit$$"dec rcx\n\t"
14852 $$emit$$"jge LOOP\n\t"
14853 $$emit$$"jmp DONE\n\t"
14854 $$emit$$"# LARGE:\n\t"
14855 if (UseXMMForObjInit) {
14856 $$emit$$"movdq $tmp, $val\n\t"
14857 $$emit$$"punpcklqdq $tmp, $tmp\n\t"
14858 $$emit$$"vinserti128_high $tmp, $tmp\n\t"
14859 $$emit$$"jmpq L_zero_64_bytes\n\t"
14860 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14861 $$emit$$"vmovdqu $tmp,(rax)\n\t"
14862 $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
14863 $$emit$$"add 0x40,rax\n\t"
14864 $$emit$$"# L_zero_64_bytes:\n\t"
14865 $$emit$$"sub 0x8,rcx\n\t"
14866 $$emit$$"jge L_loop\n\t"
14867 $$emit$$"add 0x4,rcx\n\t"
14868 $$emit$$"jl L_tail\n\t"
14869 $$emit$$"vmovdqu $tmp,(rax)\n\t"
14870 $$emit$$"add 0x20,rax\n\t"
14871 $$emit$$"sub 0x4,rcx\n\t"
14872 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14873 $$emit$$"add 0x4,rcx\n\t"
14874 $$emit$$"jle L_end\n\t"
14875 $$emit$$"dec rcx\n\t"
14876 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14877 $$emit$$"vmovq xmm0,(rax)\n\t"
14878 $$emit$$"add 0x8,rax\n\t"
14879 $$emit$$"dec rcx\n\t"
14880 $$emit$$"jge L_sloop\n\t"
14881 $$emit$$"# L_end:\n\t"
14882 } else {
14883 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14884 }
14885 $$emit$$"# DONE"
14886 %}
14887 ins_encode %{
14888 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
14889 $tmp$$XMMRegister, false, true);
14890 %}
14891 ins_pipe(pipe_slow);
14892 %}
14893
14894 // Small non-constant length ClearArray for AVX512 targets.
14895 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
14896 Universe dummy, rFlagsReg cr)
14897 %{
14898 predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
14899 match(Set dummy (ClearArray (Binary cnt base) val));
14900 ins_cost(125);
14901 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
14902
14903 format %{ $$template
14904 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14905 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14906 $$emit$$"jg LARGE\n\t"
14907 $$emit$$"dec rcx\n\t"
14908 $$emit$$"js DONE\t# Zero length\n\t"
14909 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14910 $$emit$$"dec rcx\n\t"
14911 $$emit$$"jge LOOP\n\t"
14912 $$emit$$"jmp DONE\n\t"
14913 $$emit$$"# LARGE:\n\t"
14914 if (UseFastStosb) {
14915 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14916 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14917 } else if (UseXMMForObjInit) {
14918 $$emit$$"mov rdi,rax\n\t"
14919 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14920 $$emit$$"jmpq L_zero_64_bytes\n\t"
14921 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14922 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14923 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14924 $$emit$$"add 0x40,rax\n\t"
14925 $$emit$$"# L_zero_64_bytes:\n\t"
14926 $$emit$$"sub 0x8,rcx\n\t"
14927 $$emit$$"jge L_loop\n\t"
14928 $$emit$$"add 0x4,rcx\n\t"
14929 $$emit$$"jl L_tail\n\t"
14930 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14931 $$emit$$"add 0x20,rax\n\t"
14932 $$emit$$"sub 0x4,rcx\n\t"
14933 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14934 $$emit$$"add 0x4,rcx\n\t"
14935 $$emit$$"jle L_end\n\t"
14936 $$emit$$"dec rcx\n\t"
14937 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14938 $$emit$$"vmovq xmm0,(rax)\n\t"
14939 $$emit$$"add 0x8,rax\n\t"
14940 $$emit$$"dec rcx\n\t"
14941 $$emit$$"jge L_sloop\n\t"
14942 $$emit$$"# L_end:\n\t"
14943 } else {
14944 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14945 }
14946 $$emit$$"# DONE"
14947 %}
14948 ins_encode %{
14949 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
14950 $tmp$$XMMRegister, false, false, $ktmp$$KRegister);
14951 %}
14952 ins_pipe(pipe_slow);
14953 %}
14954
14955 instruct rep_stos_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
14956 Universe dummy, rFlagsReg cr)
14957 %{
14958 predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
14959 match(Set dummy (ClearArray (Binary cnt base) val));
14960 ins_cost(125);
14961 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
14962
14963 format %{ $$template
14964 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14965 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14966 $$emit$$"jg LARGE\n\t"
14967 $$emit$$"dec rcx\n\t"
14968 $$emit$$"js DONE\t# Zero length\n\t"
14969 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14970 $$emit$$"dec rcx\n\t"
14971 $$emit$$"jge LOOP\n\t"
14972 $$emit$$"jmp DONE\n\t"
14973 $$emit$$"# LARGE:\n\t"
14974 if (UseFastStosb) {
14975 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14976 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14977 } else if (UseXMMForObjInit) {
14978 $$emit$$"mov rdi,rax\n\t"
14979 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14980 $$emit$$"jmpq L_zero_64_bytes\n\t"
14981 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14982 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14983 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14984 $$emit$$"add 0x40,rax\n\t"
14985 $$emit$$"# L_zero_64_bytes:\n\t"
14986 $$emit$$"sub 0x8,rcx\n\t"
14987 $$emit$$"jge L_loop\n\t"
14988 $$emit$$"add 0x4,rcx\n\t"
14989 $$emit$$"jl L_tail\n\t"
14990 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14991 $$emit$$"add 0x20,rax\n\t"
14992 $$emit$$"sub 0x4,rcx\n\t"
14993 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14994 $$emit$$"add 0x4,rcx\n\t"
14995 $$emit$$"jle L_end\n\t"
14996 $$emit$$"dec rcx\n\t"
14997 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14998 $$emit$$"vmovq xmm0,(rax)\n\t"
14999 $$emit$$"add 0x8,rax\n\t"
15000 $$emit$$"dec rcx\n\t"
15001 $$emit$$"jge L_sloop\n\t"
15002 $$emit$$"# L_end:\n\t"
15003 } else {
15004 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
15005 }
15006 $$emit$$"# DONE"
15007 %}
15008 ins_encode %{
15009 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15010 $tmp$$XMMRegister, false, true, $ktmp$$KRegister);
15011 %}
15012 ins_pipe(pipe_slow);
15013 %}
15014
15015 // Large non-constant length ClearArray for non-AVX512 targets.
15016 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
15017 Universe dummy, rFlagsReg cr)
15018 %{
15019 predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
15020 match(Set dummy (ClearArray (Binary cnt base) val));
15021 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
15022
15023 format %{ $$template
15024 if (UseFastStosb) {
15025 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
15026 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
15027 } else if (UseXMMForObjInit) {
15028 $$emit$$"movdq $tmp, $val\n\t"
15029 $$emit$$"punpcklqdq $tmp, $tmp\n\t"
15030 $$emit$$"vinserti128_high $tmp, $tmp\n\t"
15031 $$emit$$"jmpq L_zero_64_bytes\n\t"
15032 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
15033 $$emit$$"vmovdqu $tmp,(rax)\n\t"
15034 $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
15035 $$emit$$"add 0x40,rax\n\t"
15036 $$emit$$"# L_zero_64_bytes:\n\t"
15037 $$emit$$"sub 0x8,rcx\n\t"
15038 $$emit$$"jge L_loop\n\t"
15039 $$emit$$"add 0x4,rcx\n\t"
15040 $$emit$$"jl L_tail\n\t"
15041 $$emit$$"vmovdqu $tmp,(rax)\n\t"
15042 $$emit$$"add 0x20,rax\n\t"
15043 $$emit$$"sub 0x4,rcx\n\t"
15044 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
15045 $$emit$$"add 0x4,rcx\n\t"
15046 $$emit$$"jle L_end\n\t"
15047 $$emit$$"dec rcx\n\t"
15048 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
15049 $$emit$$"vmovq xmm0,(rax)\n\t"
15050 $$emit$$"add 0x8,rax\n\t"
15051 $$emit$$"dec rcx\n\t"
15052 $$emit$$"jge L_sloop\n\t"
15053 $$emit$$"# L_end:\n\t"
15054 } else {
15055 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
15056 }
15057 %}
15058 ins_encode %{
15059 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15060 $tmp$$XMMRegister, true, false);
15061 %}
15062 ins_pipe(pipe_slow);
15063 %}
15064
15065 instruct rep_stos_large_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
15066 Universe dummy, rFlagsReg cr)
15067 %{
15068 predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
15069 match(Set dummy (ClearArray (Binary cnt base) val));
15070 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
15071
15072 format %{ $$template
15073 if (UseXMMForObjInit) {
15074 $$emit$$"movdq $tmp, $val\n\t"
15075 $$emit$$"punpcklqdq $tmp, $tmp\n\t"
15076 $$emit$$"vinserti128_high $tmp, $tmp\n\t"
15077 $$emit$$"jmpq L_zero_64_bytes\n\t"
15078 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
15079 $$emit$$"vmovdqu $tmp,(rax)\n\t"
15080 $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
15081 $$emit$$"add 0x40,rax\n\t"
15082 $$emit$$"# L_zero_64_bytes:\n\t"
15083 $$emit$$"sub 0x8,rcx\n\t"
15084 $$emit$$"jge L_loop\n\t"
15085 $$emit$$"add 0x4,rcx\n\t"
15086 $$emit$$"jl L_tail\n\t"
15087 $$emit$$"vmovdqu $tmp,(rax)\n\t"
15088 $$emit$$"add 0x20,rax\n\t"
15089 $$emit$$"sub 0x4,rcx\n\t"
15090 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
15091 $$emit$$"add 0x4,rcx\n\t"
15092 $$emit$$"jle L_end\n\t"
15093 $$emit$$"dec rcx\n\t"
15094 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
15095 $$emit$$"vmovq xmm0,(rax)\n\t"
15096 $$emit$$"add 0x8,rax\n\t"
15097 $$emit$$"dec rcx\n\t"
15098 $$emit$$"jge L_sloop\n\t"
15099 $$emit$$"# L_end:\n\t"
15100 } else {
15101 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
15102 }
15103 %}
15104 ins_encode %{
15105 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15106 $tmp$$XMMRegister, true, true);
15107 %}
15108 ins_pipe(pipe_slow);
15109 %}
15110
15111 // Large non-constant length ClearArray for AVX512 targets.
15112 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
15113 Universe dummy, rFlagsReg cr)
15114 %{
15115 predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
15116 match(Set dummy (ClearArray (Binary cnt base) val));
15117 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
15118
15119 format %{ $$template
15120 if (UseFastStosb) {
15121 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
15122 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
15123 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
15124 } else if (UseXMMForObjInit) {
15125 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
15126 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
15127 $$emit$$"jmpq L_zero_64_bytes\n\t"
15128 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
15129 $$emit$$"vmovdqu ymm0,(rax)\n\t"
15130 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
15131 $$emit$$"add 0x40,rax\n\t"
15132 $$emit$$"# L_zero_64_bytes:\n\t"
15133 $$emit$$"sub 0x8,rcx\n\t"
15134 $$emit$$"jge L_loop\n\t"
15135 $$emit$$"add 0x4,rcx\n\t"
15136 $$emit$$"jl L_tail\n\t"
15137 $$emit$$"vmovdqu ymm0,(rax)\n\t"
15138 $$emit$$"add 0x20,rax\n\t"
15139 $$emit$$"sub 0x4,rcx\n\t"
15140 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
15141 $$emit$$"add 0x4,rcx\n\t"
15142 $$emit$$"jle L_end\n\t"
15143 $$emit$$"dec rcx\n\t"
15144 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
15145 $$emit$$"vmovq xmm0,(rax)\n\t"
15146 $$emit$$"add 0x8,rax\n\t"
15147 $$emit$$"dec rcx\n\t"
15148 $$emit$$"jge L_sloop\n\t"
15149 $$emit$$"# L_end:\n\t"
15150 } else {
15151 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
15152 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
15153 }
15154 %}
15155 ins_encode %{
15156 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15157 $tmp$$XMMRegister, true, false, $ktmp$$KRegister);
15158 %}
15159 ins_pipe(pipe_slow);
15160 %}
15161
15162 instruct rep_stos_large_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
15163 Universe dummy, rFlagsReg cr)
15164 %{
15165 predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
15166 match(Set dummy (ClearArray (Binary cnt base) val));
15167 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
15168
15169 format %{ $$template
15170 if (UseFastStosb) {
15171 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
15172 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
15173 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
15174 } else if (UseXMMForObjInit) {
15175 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
15176 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
15177 $$emit$$"jmpq L_zero_64_bytes\n\t"
15178 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
15179 $$emit$$"vmovdqu ymm0,(rax)\n\t"
15180 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
15181 $$emit$$"add 0x40,rax\n\t"
15182 $$emit$$"# L_zero_64_bytes:\n\t"
15183 $$emit$$"sub 0x8,rcx\n\t"
15184 $$emit$$"jge L_loop\n\t"
15185 $$emit$$"add 0x4,rcx\n\t"
15186 $$emit$$"jl L_tail\n\t"
15187 $$emit$$"vmovdqu ymm0,(rax)\n\t"
15188 $$emit$$"add 0x20,rax\n\t"
15189 $$emit$$"sub 0x4,rcx\n\t"
15190 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
15191 $$emit$$"add 0x4,rcx\n\t"
15192 $$emit$$"jle L_end\n\t"
15193 $$emit$$"dec rcx\n\t"
15194 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
15195 $$emit$$"vmovq xmm0,(rax)\n\t"
15196 $$emit$$"add 0x8,rax\n\t"
15197 $$emit$$"dec rcx\n\t"
15198 $$emit$$"jge L_sloop\n\t"
15199 $$emit$$"# L_end:\n\t"
15200 } else {
15201 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
15202 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
15203 }
15204 %}
15205 ins_encode %{
15206 __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
15207 $tmp$$XMMRegister, true, true, $ktmp$$KRegister);
15208 %}
15209 ins_pipe(pipe_slow);
15210 %}
15211
15212 // Small constant length ClearArray for AVX512 targets.
15213 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rax_RegL val, kReg ktmp, Universe dummy, rFlagsReg cr)
15214 %{
15215 predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() &&
15216 ((MaxVectorSize >= 32) && VM_Version::supports_avx512vl()));
15217 match(Set dummy (ClearArray (Binary cnt base) val));
15218 ins_cost(100);
15219 effect(TEMP tmp, USE_KILL val, TEMP ktmp, KILL cr);
15220 format %{ "clear_mem_imm $base , $cnt \n\t" %}
15221 ins_encode %{
15222 __ clear_mem($base$$Register, $cnt$$constant, $val$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
15223 %}
15224 ins_pipe(pipe_slow);
15225 %}
15226
15227 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
15228 rax_RegI result, legRegD tmp1, rFlagsReg cr)
15229 %{
15230 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
15231 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15232 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15233
15234 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15235 ins_encode %{
15236 __ string_compare($str1$$Register, $str2$$Register,
15237 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15238 $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
15239 %}
15240 ins_pipe( pipe_slow );
15241 %}
15242
15243 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
15244 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
15245 %{
15246 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
15247 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15248 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15249
15250 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15251 ins_encode %{
15252 __ string_compare($str1$$Register, $str2$$Register,
15253 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15254 $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
15255 %}
15256 ins_pipe( pipe_slow );
15257 %}
15258
15259 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
15260 rax_RegI result, legRegD tmp1, rFlagsReg cr)
15261 %{
15262 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
15263 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15264 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15265
15266 format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15267 ins_encode %{
15268 __ string_compare($str1$$Register, $str2$$Register,
15269 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15270 $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
15271 %}
15272 ins_pipe( pipe_slow );
15273 %}
15274
15275 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
15276 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
15277 %{
15278 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
15279 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15280 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15281
15282 format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15283 ins_encode %{
15284 __ string_compare($str1$$Register, $str2$$Register,
15285 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15286 $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
15287 %}
15288 ins_pipe( pipe_slow );
15289 %}
15290
15291 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
15292 rax_RegI result, legRegD tmp1, rFlagsReg cr)
15293 %{
15294 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
15295 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15296 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15297
15298 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15299 ins_encode %{
15300 __ string_compare($str1$$Register, $str2$$Register,
15301 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15302 $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
15303 %}
15304 ins_pipe( pipe_slow );
15305 %}
15306
15307 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
15308 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
15309 %{
15310 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
15311 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15312 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15313
15314 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15315 ins_encode %{
15316 __ string_compare($str1$$Register, $str2$$Register,
15317 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15318 $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
15319 %}
15320 ins_pipe( pipe_slow );
15321 %}
15322
15323 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
15324 rax_RegI result, legRegD tmp1, rFlagsReg cr)
15325 %{
15326 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
15327 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15328 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15329
15330 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15331 ins_encode %{
15332 __ string_compare($str2$$Register, $str1$$Register,
15333 $cnt2$$Register, $cnt1$$Register, $result$$Register,
15334 $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
15335 %}
15336 ins_pipe( pipe_slow );
15337 %}
15338
15339 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
15340 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
15341 %{
15342 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
15343 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15344 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15345
15346 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15347 ins_encode %{
15348 __ string_compare($str2$$Register, $str1$$Register,
15349 $cnt2$$Register, $cnt1$$Register, $result$$Register,
15350 $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
15351 %}
15352 ins_pipe( pipe_slow );
15353 %}
15354
15355 // fast search of substring with known size.
15356 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
15357 rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
15358 %{
15359 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
15360 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15361 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
15362
15363 format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
15364 ins_encode %{
15365 int icnt2 = (int)$int_cnt2$$constant;
15366 if (icnt2 >= 16) {
15367 // IndexOf for constant substrings with size >= 16 elements
15368 // which don't need to be loaded through stack.
15369 __ string_indexofC8($str1$$Register, $str2$$Register,
15370 $cnt1$$Register, $cnt2$$Register,
15371 icnt2, $result$$Register,
15372 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
15373 } else {
15374 // Small strings are loaded through stack if they cross page boundary.
15375 __ string_indexof($str1$$Register, $str2$$Register,
15376 $cnt1$$Register, $cnt2$$Register,
15377 icnt2, $result$$Register,
15378 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
15379 }
15380 %}
15381 ins_pipe( pipe_slow );
15382 %}
15383
15384 // fast search of substring with known size.
15385 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
15386 rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
15387 %{
15388 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
15389 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15390 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
15391
15392 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
15393 ins_encode %{
15394 int icnt2 = (int)$int_cnt2$$constant;
15395 if (icnt2 >= 8) {
15396 // IndexOf for constant substrings with size >= 8 elements
15397 // which don't need to be loaded through stack.
15398 __ string_indexofC8($str1$$Register, $str2$$Register,
15399 $cnt1$$Register, $cnt2$$Register,
15400 icnt2, $result$$Register,
15401 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
15402 } else {
15403 // Small strings are loaded through stack if they cross page boundary.
15404 __ string_indexof($str1$$Register, $str2$$Register,
15405 $cnt1$$Register, $cnt2$$Register,
15406 icnt2, $result$$Register,
15407 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
15408 }
15409 %}
15410 ins_pipe( pipe_slow );
15411 %}
15412
15413 // fast search of substring with known size.
15414 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
15415 rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
15416 %{
15417 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
15418 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15419 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
15420
15421 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
15422 ins_encode %{
15423 int icnt2 = (int)$int_cnt2$$constant;
15424 if (icnt2 >= 8) {
15425 // IndexOf for constant substrings with size >= 8 elements
15426 // which don't need to be loaded through stack.
15427 __ string_indexofC8($str1$$Register, $str2$$Register,
15428 $cnt1$$Register, $cnt2$$Register,
15429 icnt2, $result$$Register,
15430 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
15431 } else {
15432 // Small strings are loaded through stack if they cross page boundary.
15433 __ string_indexof($str1$$Register, $str2$$Register,
15434 $cnt1$$Register, $cnt2$$Register,
15435 icnt2, $result$$Register,
15436 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
15437 }
15438 %}
15439 ins_pipe( pipe_slow );
15440 %}
15441
15442 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
15443 rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
15444 %{
15445 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
15446 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15447 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
15448
15449 format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
15450 ins_encode %{
15451 __ string_indexof($str1$$Register, $str2$$Register,
15452 $cnt1$$Register, $cnt2$$Register,
15453 (-1), $result$$Register,
15454 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
15455 %}
15456 ins_pipe( pipe_slow );
15457 %}
15458
15459 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
15460 rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
15461 %{
15462 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
15463 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15464 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
15465
15466 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
15467 ins_encode %{
15468 __ string_indexof($str1$$Register, $str2$$Register,
15469 $cnt1$$Register, $cnt2$$Register,
15470 (-1), $result$$Register,
15471 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
15472 %}
15473 ins_pipe( pipe_slow );
15474 %}
15475
15476 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
15477 rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
15478 %{
15479 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
15480 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15481 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
15482
15483 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
15484 ins_encode %{
15485 __ string_indexof($str1$$Register, $str2$$Register,
15486 $cnt1$$Register, $cnt2$$Register,
15487 (-1), $result$$Register,
15488 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
15489 %}
15490 ins_pipe( pipe_slow );
15491 %}
15492
15493 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
15494 rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
15495 %{
15496 predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15497 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15498 effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
15499 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result // KILL all" %}
15500 ins_encode %{
15501 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
15502 $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
15503 %}
15504 ins_pipe( pipe_slow );
15505 %}
15506
15507 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
15508 rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
15509 %{
15510 predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15511 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15512 effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
15513 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result // KILL all" %}
15514 ins_encode %{
15515 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
15516 $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
15517 %}
15518 ins_pipe( pipe_slow );
15519 %}
15520
15521 // fast string equals
15522 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
15523 legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
15524 %{
15525 predicate(!VM_Version::supports_avx512vlbw());
15526 match(Set result (StrEquals (Binary str1 str2) cnt));
15527 effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
15528
15529 format %{ "String Equals $str1,$str2,$cnt -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15530 ins_encode %{
15531 __ arrays_equals(false, $str1$$Register, $str2$$Register,
15532 $cnt$$Register, $result$$Register, $tmp3$$Register,
15533 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
15534 %}
15535 ins_pipe( pipe_slow );
15536 %}
15537
15538 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
15539 legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
15540 %{
15541 predicate(VM_Version::supports_avx512vlbw());
15542 match(Set result (StrEquals (Binary str1 str2) cnt));
15543 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
15544
15545 format %{ "String Equals $str1,$str2,$cnt -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15546 ins_encode %{
15547 __ arrays_equals(false, $str1$$Register, $str2$$Register,
15548 $cnt$$Register, $result$$Register, $tmp3$$Register,
15549 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
15550 %}
15551 ins_pipe( pipe_slow );
15552 %}
15553
15554 // fast array equals
15555 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15556 legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15557 %{
15558 predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
15559 match(Set result (AryEq ary1 ary2));
15560 effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15561
15562 format %{ "Array Equals byte[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15563 ins_encode %{
15564 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15565 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15566 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
15567 %}
15568 ins_pipe( pipe_slow );
15569 %}
15570
15571 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15572 legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15573 %{
15574 predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
15575 match(Set result (AryEq ary1 ary2));
15576 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15577
15578 format %{ "Array Equals byte[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15579 ins_encode %{
15580 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15581 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15582 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
15583 %}
15584 ins_pipe( pipe_slow );
15585 %}
15586
15587 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15588 legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15589 %{
15590 predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
15591 match(Set result (AryEq ary1 ary2));
15592 effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15593
15594 format %{ "Array Equals char[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15595 ins_encode %{
15596 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15597 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15598 $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
15599 %}
15600 ins_pipe( pipe_slow );
15601 %}
15602
15603 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15604 legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15605 %{
15606 predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
15607 match(Set result (AryEq ary1 ary2));
15608 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15609
15610 format %{ "Array Equals char[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15611 ins_encode %{
15612 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15613 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15614 $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
15615 %}
15616 ins_pipe( pipe_slow );
15617 %}
15618
15619 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
15620 legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
15621 legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
15622 legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
15623 legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
15624 %{
15625 predicate(UseAVX >= 2);
15626 match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
15627 effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
15628 TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
15629 TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
15630 USE basic_type, KILL cr);
15631
15632 format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result // KILL all" %}
15633 ins_encode %{
15634 __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
15635 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
15636 $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
15637 $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
15638 $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
15639 $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
15640 $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
15641 %}
15642 ins_pipe( pipe_slow );
15643 %}
15644
15645 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
15646 legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
15647 %{
15648 predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
15649 match(Set result (CountPositives ary1 len));
15650 effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
15651
15652 format %{ "countPositives byte[] $ary1,$len -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15653 ins_encode %{
15654 __ count_positives($ary1$$Register, $len$$Register,
15655 $result$$Register, $tmp3$$Register,
15656 $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
15657 %}
15658 ins_pipe( pipe_slow );
15659 %}
15660
15661 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
15662 legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
15663 %{
15664 predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
15665 match(Set result (CountPositives ary1 len));
15666 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
15667
15668 format %{ "countPositives byte[] $ary1,$len -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15669 ins_encode %{
15670 __ count_positives($ary1$$Register, $len$$Register,
15671 $result$$Register, $tmp3$$Register,
15672 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
15673 %}
15674 ins_pipe( pipe_slow );
15675 %}
15676
15677 // fast char[] to byte[] compression
15678 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
15679 legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15680 predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
15681 match(Set result (StrCompressedCopy src (Binary dst len)));
15682 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
15683 USE_KILL len, KILL tmp5, KILL cr);
15684
15685 format %{ "String Compress $src,$dst -> $result // KILL RAX, RCX, RDX" %}
15686 ins_encode %{
15687 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
15688 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15689 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
15690 knoreg, knoreg);
15691 %}
15692 ins_pipe( pipe_slow );
15693 %}
15694
15695 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
15696 legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15697 predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
15698 match(Set result (StrCompressedCopy src (Binary dst len)));
15699 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
15700 USE_KILL len, KILL tmp5, KILL cr);
15701
15702 format %{ "String Compress $src,$dst -> $result // KILL RAX, RCX, RDX" %}
15703 ins_encode %{
15704 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
15705 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15706 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
15707 $ktmp1$$KRegister, $ktmp2$$KRegister);
15708 %}
15709 ins_pipe( pipe_slow );
15710 %}
15711 // fast byte[] to char[] inflation
15712 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15713 legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
15714 predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
15715 match(Set dummy (StrInflatedCopy src (Binary dst len)));
15716 effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
15717
15718 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
15719 ins_encode %{
15720 __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
15721 $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
15722 %}
15723 ins_pipe( pipe_slow );
15724 %}
15725
15726 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15727 legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
15728 predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
15729 match(Set dummy (StrInflatedCopy src (Binary dst len)));
15730 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
15731
15732 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
15733 ins_encode %{
15734 __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
15735 $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
15736 %}
15737 ins_pipe( pipe_slow );
15738 %}
15739
15740 // encode char[] to byte[] in ISO_8859_1
15741 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15742 legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
15743 rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15744 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
15745 match(Set result (EncodeISOArray src (Binary dst len)));
15746 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
15747
15748 format %{ "Encode iso array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
15749 ins_encode %{
15750 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
15751 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15752 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
15753 %}
15754 ins_pipe( pipe_slow );
15755 %}
15756
15757 // encode char[] to byte[] in ASCII
15758 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15759 legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
15760 rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15761 predicate(((EncodeISOArrayNode*)n)->is_ascii());
15762 match(Set result (EncodeISOArray src (Binary dst len)));
15763 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
15764
15765 format %{ "Encode ascii array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
15766 ins_encode %{
15767 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
15768 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15769 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
15770 %}
15771 ins_pipe( pipe_slow );
15772 %}
15773
15774 //----------Overflow Math Instructions-----------------------------------------
15775
15776 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
15777 %{
15778 match(Set cr (OverflowAddI op1 op2));
15779 effect(DEF cr, USE_KILL op1, USE op2);
15780
15781 format %{ "addl $op1, $op2\t# overflow check int" %}
15782
15783 ins_encode %{
15784 __ addl($op1$$Register, $op2$$Register);
15785 %}
15786 ins_pipe(ialu_reg_reg);
15787 %}
15788
15789 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
15790 %{
15791 match(Set cr (OverflowAddI op1 op2));
15792 effect(DEF cr, USE_KILL op1, USE op2);
15793
15794 format %{ "addl $op1, $op2\t# overflow check int" %}
15795
15796 ins_encode %{
15797 __ addl($op1$$Register, $op2$$constant);
15798 %}
15799 ins_pipe(ialu_reg_reg);
15800 %}
15801
15802 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
15803 %{
15804 match(Set cr (OverflowAddL op1 op2));
15805 effect(DEF cr, USE_KILL op1, USE op2);
15806
15807 format %{ "addq $op1, $op2\t# overflow check long" %}
15808 ins_encode %{
15809 __ addq($op1$$Register, $op2$$Register);
15810 %}
15811 ins_pipe(ialu_reg_reg);
15812 %}
15813
15814 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
15815 %{
15816 match(Set cr (OverflowAddL op1 op2));
15817 effect(DEF cr, USE_KILL op1, USE op2);
15818
15819 format %{ "addq $op1, $op2\t# overflow check long" %}
15820 ins_encode %{
15821 __ addq($op1$$Register, $op2$$constant);
15822 %}
15823 ins_pipe(ialu_reg_reg);
15824 %}
15825
15826 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
15827 %{
15828 match(Set cr (OverflowSubI op1 op2));
15829
15830 format %{ "cmpl $op1, $op2\t# overflow check int" %}
15831 ins_encode %{
15832 __ cmpl($op1$$Register, $op2$$Register);
15833 %}
15834 ins_pipe(ialu_reg_reg);
15835 %}
15836
15837 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
15838 %{
15839 match(Set cr (OverflowSubI op1 op2));
15840
15841 format %{ "cmpl $op1, $op2\t# overflow check int" %}
15842 ins_encode %{
15843 __ cmpl($op1$$Register, $op2$$constant);
15844 %}
15845 ins_pipe(ialu_reg_reg);
15846 %}
15847
15848 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
15849 %{
15850 match(Set cr (OverflowSubL op1 op2));
15851
15852 format %{ "cmpq $op1, $op2\t# overflow check long" %}
15853 ins_encode %{
15854 __ cmpq($op1$$Register, $op2$$Register);
15855 %}
15856 ins_pipe(ialu_reg_reg);
15857 %}
15858
15859 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
15860 %{
15861 match(Set cr (OverflowSubL op1 op2));
15862
15863 format %{ "cmpq $op1, $op2\t# overflow check long" %}
15864 ins_encode %{
15865 __ cmpq($op1$$Register, $op2$$constant);
15866 %}
15867 ins_pipe(ialu_reg_reg);
15868 %}
15869
15870 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
15871 %{
15872 match(Set cr (OverflowSubI zero op2));
15873 effect(DEF cr, USE_KILL op2);
15874
15875 format %{ "negl $op2\t# overflow check int" %}
15876 ins_encode %{
15877 __ negl($op2$$Register);
15878 %}
15879 ins_pipe(ialu_reg_reg);
15880 %}
15881
15882 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
15883 %{
15884 match(Set cr (OverflowSubL zero op2));
15885 effect(DEF cr, USE_KILL op2);
15886
15887 format %{ "negq $op2\t# overflow check long" %}
15888 ins_encode %{
15889 __ negq($op2$$Register);
15890 %}
15891 ins_pipe(ialu_reg_reg);
15892 %}
15893
15894 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
15895 %{
15896 match(Set cr (OverflowMulI op1 op2));
15897 effect(DEF cr, USE_KILL op1, USE op2);
15898
15899 format %{ "imull $op1, $op2\t# overflow check int" %}
15900 ins_encode %{
15901 __ imull($op1$$Register, $op2$$Register);
15902 %}
15903 ins_pipe(ialu_reg_reg_alu0);
15904 %}
15905
15906 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
15907 %{
15908 match(Set cr (OverflowMulI op1 op2));
15909 effect(DEF cr, TEMP tmp, USE op1, USE op2);
15910
15911 format %{ "imull $tmp, $op1, $op2\t# overflow check int" %}
15912 ins_encode %{
15913 __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
15914 %}
15915 ins_pipe(ialu_reg_reg_alu0);
15916 %}
15917
15918 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
15919 %{
15920 match(Set cr (OverflowMulL op1 op2));
15921 effect(DEF cr, USE_KILL op1, USE op2);
15922
15923 format %{ "imulq $op1, $op2\t# overflow check long" %}
15924 ins_encode %{
15925 __ imulq($op1$$Register, $op2$$Register);
15926 %}
15927 ins_pipe(ialu_reg_reg_alu0);
15928 %}
15929
15930 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
15931 %{
15932 match(Set cr (OverflowMulL op1 op2));
15933 effect(DEF cr, TEMP tmp, USE op1, USE op2);
15934
15935 format %{ "imulq $tmp, $op1, $op2\t# overflow check long" %}
15936 ins_encode %{
15937 __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
15938 %}
15939 ins_pipe(ialu_reg_reg_alu0);
15940 %}
15941
15942
15943 //----------Control Flow Instructions------------------------------------------
15944 // Signed compare Instructions
15945
15946 // XXX more variants!!
15947 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
15948 %{
15949 match(Set cr (CmpI op1 op2));
15950 effect(DEF cr, USE op1, USE op2);
15951
15952 format %{ "cmpl $op1, $op2" %}
15953 ins_encode %{
15954 __ cmpl($op1$$Register, $op2$$Register);
15955 %}
15956 ins_pipe(ialu_cr_reg_reg);
15957 %}
15958
15959 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
15960 %{
15961 match(Set cr (CmpI op1 op2));
15962
15963 format %{ "cmpl $op1, $op2" %}
15964 ins_encode %{
15965 __ cmpl($op1$$Register, $op2$$constant);
15966 %}
15967 ins_pipe(ialu_cr_reg_imm);
15968 %}
15969
15970 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
15971 %{
15972 match(Set cr (CmpI op1 (LoadI op2)));
15973
15974 ins_cost(500); // XXX
15975 format %{ "cmpl $op1, $op2" %}
15976 ins_encode %{
15977 __ cmpl($op1$$Register, $op2$$Address);
15978 %}
15979 ins_pipe(ialu_cr_reg_mem);
15980 %}
15981
15982 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
15983 %{
15984 match(Set cr (CmpI src zero));
15985
15986 format %{ "testl $src, $src" %}
15987 ins_encode %{
15988 __ testl($src$$Register, $src$$Register);
15989 %}
15990 ins_pipe(ialu_cr_reg_imm);
15991 %}
15992
15993 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
15994 %{
15995 match(Set cr (CmpI (AndI src con) zero));
15996
15997 format %{ "testl $src, $con" %}
15998 ins_encode %{
15999 __ testl($src$$Register, $con$$constant);
16000 %}
16001 ins_pipe(ialu_cr_reg_imm);
16002 %}
16003
16004 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
16005 %{
16006 match(Set cr (CmpI (AndI src1 src2) zero));
16007
16008 format %{ "testl $src1, $src2" %}
16009 ins_encode %{
16010 __ testl($src1$$Register, $src2$$Register);
16011 %}
16012 ins_pipe(ialu_cr_reg_imm);
16013 %}
16014
16015 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
16016 %{
16017 match(Set cr (CmpI (AndI src (LoadI mem)) zero));
16018
16019 format %{ "testl $src, $mem" %}
16020 ins_encode %{
16021 __ testl($src$$Register, $mem$$Address);
16022 %}
16023 ins_pipe(ialu_cr_reg_mem);
16024 %}
16025
16026 // Unsigned compare Instructions; really, same as signed except they
16027 // produce an rFlagsRegU instead of rFlagsReg.
16028 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
16029 %{
16030 match(Set cr (CmpU op1 op2));
16031
16032 format %{ "cmpl $op1, $op2\t# unsigned" %}
16033 ins_encode %{
16034 __ cmpl($op1$$Register, $op2$$Register);
16035 %}
16036 ins_pipe(ialu_cr_reg_reg);
16037 %}
16038
16039 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
16040 %{
16041 match(Set cr (CmpU op1 op2));
16042
16043 format %{ "cmpl $op1, $op2\t# unsigned" %}
16044 ins_encode %{
16045 __ cmpl($op1$$Register, $op2$$constant);
16046 %}
16047 ins_pipe(ialu_cr_reg_imm);
16048 %}
16049
16050 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
16051 %{
16052 match(Set cr (CmpU op1 (LoadI op2)));
16053
16054 ins_cost(500); // XXX
16055 format %{ "cmpl $op1, $op2\t# unsigned" %}
16056 ins_encode %{
16057 __ cmpl($op1$$Register, $op2$$Address);
16058 %}
16059 ins_pipe(ialu_cr_reg_mem);
16060 %}
16061
16062 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
16063 %{
16064 match(Set cr (CmpU src zero));
16065
16066 format %{ "testl $src, $src\t# unsigned" %}
16067 ins_encode %{
16068 __ testl($src$$Register, $src$$Register);
16069 %}
16070 ins_pipe(ialu_cr_reg_imm);
16071 %}
16072
16073 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
16074 %{
16075 match(Set cr (CmpP op1 op2));
16076
16077 format %{ "cmpq $op1, $op2\t# ptr" %}
16078 ins_encode %{
16079 __ cmpq($op1$$Register, $op2$$Register);
16080 %}
16081 ins_pipe(ialu_cr_reg_reg);
16082 %}
16083
16084 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
16085 %{
16086 match(Set cr (CmpP op1 (LoadP op2)));
16087 predicate(n->in(2)->as_Load()->barrier_data() == 0);
16088
16089 ins_cost(500); // XXX
16090 format %{ "cmpq $op1, $op2\t# ptr" %}
16091 ins_encode %{
16092 __ cmpq($op1$$Register, $op2$$Address);
16093 %}
16094 ins_pipe(ialu_cr_reg_mem);
16095 %}
16096
16097 // XXX this is generalized by compP_rReg_mem???
16098 // Compare raw pointer (used in out-of-heap check).
16099 // Only works because non-oop pointers must be raw pointers
16100 // and raw pointers have no anti-dependencies.
16101 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
16102 %{
16103 predicate(n->in(2)->in(2)->bottom_type()->isa_rawptr() != nullptr &&
16104 n->in(2)->as_Load()->barrier_data() == 0);
16105 match(Set cr (CmpP op1 (LoadP op2)));
16106
16107 format %{ "cmpq $op1, $op2\t# raw ptr" %}
16108 ins_encode %{
16109 __ cmpq($op1$$Register, $op2$$Address);
16110 %}
16111 ins_pipe(ialu_cr_reg_mem);
16112 %}
16113
16114 // This will generate a signed flags result. This should be OK since
16115 // any compare to a zero should be eq/neq.
16116 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
16117 %{
16118 match(Set cr (CmpP src zero));
16119
16120 format %{ "testq $src, $src\t# ptr" %}
16121 ins_encode %{
16122 __ testq($src$$Register, $src$$Register);
16123 %}
16124 ins_pipe(ialu_cr_reg_imm);
16125 %}
16126
16127 // This will generate a signed flags result. This should be OK since
16128 // any compare to a zero should be eq/neq.
16129 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
16130 %{
16131 predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
16132 n->in(1)->as_Load()->barrier_data() == 0);
16133 match(Set cr (CmpP (LoadP op) zero));
16134
16135 ins_cost(500); // XXX
16136 format %{ "testq $op, 0xffffffffffffffff\t# ptr" %}
16137 ins_encode %{
16138 __ testq($op$$Address, 0xFFFFFFFF);
16139 %}
16140 ins_pipe(ialu_cr_reg_imm);
16141 %}
16142
16143 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
16144 %{
16145 predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
16146 n->in(1)->as_Load()->barrier_data() == 0);
16147 match(Set cr (CmpP (LoadP mem) zero));
16148
16149 format %{ "cmpq R12, $mem\t# ptr (R12_heapbase==0)" %}
16150 ins_encode %{
16151 __ cmpq(r12, $mem$$Address);
16152 %}
16153 ins_pipe(ialu_cr_reg_mem);
16154 %}
16155
16156 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
16157 %{
16158 match(Set cr (CmpN op1 op2));
16159
16160 format %{ "cmpl $op1, $op2\t# compressed ptr" %}
16161 ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
16162 ins_pipe(ialu_cr_reg_reg);
16163 %}
16164
16165 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
16166 %{
16167 predicate(n->in(2)->as_Load()->barrier_data() == 0);
16168 match(Set cr (CmpN src (LoadN mem)));
16169
16170 format %{ "cmpl $src, $mem\t# compressed ptr" %}
16171 ins_encode %{
16172 __ cmpl($src$$Register, $mem$$Address);
16173 %}
16174 ins_pipe(ialu_cr_reg_mem);
16175 %}
16176
16177 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
16178 match(Set cr (CmpN op1 op2));
16179
16180 format %{ "cmpl $op1, $op2\t# compressed ptr" %}
16181 ins_encode %{
16182 __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
16183 %}
16184 ins_pipe(ialu_cr_reg_imm);
16185 %}
16186
16187 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
16188 %{
16189 predicate(n->in(2)->as_Load()->barrier_data() == 0);
16190 match(Set cr (CmpN src (LoadN mem)));
16191
16192 format %{ "cmpl $mem, $src\t# compressed ptr" %}
16193 ins_encode %{
16194 __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
16195 %}
16196 ins_pipe(ialu_cr_reg_mem);
16197 %}
16198
16199 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
16200 match(Set cr (CmpN op1 op2));
16201
16202 format %{ "cmpl $op1, $op2\t# compressed klass ptr" %}
16203 ins_encode %{
16204 __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
16205 %}
16206 ins_pipe(ialu_cr_reg_imm);
16207 %}
16208
16209 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
16210 %{
16211 predicate(!UseCompactObjectHeaders);
16212 match(Set cr (CmpN src (LoadNKlass mem)));
16213
16214 format %{ "cmpl $mem, $src\t# compressed klass ptr" %}
16215 ins_encode %{
16216 __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
16217 %}
16218 ins_pipe(ialu_cr_reg_mem);
16219 %}
16220
16221 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
16222 match(Set cr (CmpN src zero));
16223
16224 format %{ "testl $src, $src\t# compressed ptr" %}
16225 ins_encode %{ __ testl($src$$Register, $src$$Register); %}
16226 ins_pipe(ialu_cr_reg_imm);
16227 %}
16228
16229 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
16230 %{
16231 predicate(CompressedOops::base() != nullptr &&
16232 n->in(1)->as_Load()->barrier_data() == 0);
16233 match(Set cr (CmpN (LoadN mem) zero));
16234
16235 ins_cost(500); // XXX
16236 format %{ "testl $mem, 0xffffffff\t# compressed ptr" %}
16237 ins_encode %{
16238 __ cmpl($mem$$Address, (int)0xFFFFFFFF);
16239 %}
16240 ins_pipe(ialu_cr_reg_mem);
16241 %}
16242
16243 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
16244 %{
16245 predicate(CompressedOops::base() == nullptr &&
16246 n->in(1)->as_Load()->barrier_data() == 0);
16247 match(Set cr (CmpN (LoadN mem) zero));
16248
16249 format %{ "cmpl R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
16250 ins_encode %{
16251 __ cmpl(r12, $mem$$Address);
16252 %}
16253 ins_pipe(ialu_cr_reg_mem);
16254 %}
16255
16256 // Yanked all unsigned pointer compare operations.
16257 // Pointer compares are done with CmpP which is already unsigned.
16258
16259 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
16260 %{
16261 match(Set cr (CmpL op1 op2));
16262
16263 format %{ "cmpq $op1, $op2" %}
16264 ins_encode %{
16265 __ cmpq($op1$$Register, $op2$$Register);
16266 %}
16267 ins_pipe(ialu_cr_reg_reg);
16268 %}
16269
16270 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
16271 %{
16272 match(Set cr (CmpL op1 op2));
16273
16274 format %{ "cmpq $op1, $op2" %}
16275 ins_encode %{
16276 __ cmpq($op1$$Register, $op2$$constant);
16277 %}
16278 ins_pipe(ialu_cr_reg_imm);
16279 %}
16280
16281 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
16282 %{
16283 match(Set cr (CmpL op1 (LoadL op2)));
16284
16285 format %{ "cmpq $op1, $op2" %}
16286 ins_encode %{
16287 __ cmpq($op1$$Register, $op2$$Address);
16288 %}
16289 ins_pipe(ialu_cr_reg_mem);
16290 %}
16291
16292 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
16293 %{
16294 match(Set cr (CmpL src zero));
16295
16296 format %{ "testq $src, $src" %}
16297 ins_encode %{
16298 __ testq($src$$Register, $src$$Register);
16299 %}
16300 ins_pipe(ialu_cr_reg_imm);
16301 %}
16302
16303 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
16304 %{
16305 match(Set cr (CmpL (AndL src con) zero));
16306
16307 format %{ "testq $src, $con\t# long" %}
16308 ins_encode %{
16309 __ testq($src$$Register, $con$$constant);
16310 %}
16311 ins_pipe(ialu_cr_reg_imm);
16312 %}
16313
16314 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
16315 %{
16316 match(Set cr (CmpL (AndL src1 src2) zero));
16317
16318 format %{ "testq $src1, $src2\t# long" %}
16319 ins_encode %{
16320 __ testq($src1$$Register, $src2$$Register);
16321 %}
16322 ins_pipe(ialu_cr_reg_imm);
16323 %}
16324
16325 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
16326 %{
16327 match(Set cr (CmpL (AndL src (LoadL mem)) zero));
16328
16329 format %{ "testq $src, $mem" %}
16330 ins_encode %{
16331 __ testq($src$$Register, $mem$$Address);
16332 %}
16333 ins_pipe(ialu_cr_reg_mem);
16334 %}
16335
16336 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
16337 %{
16338 match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
16339
16340 format %{ "testq $src, $mem" %}
16341 ins_encode %{
16342 __ testq($src$$Register, $mem$$Address);
16343 %}
16344 ins_pipe(ialu_cr_reg_mem);
16345 %}
16346
16347 // Manifest a CmpU result in an integer register. Very painful.
16348 // This is the test to avoid.
16349 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
16350 %{
16351 match(Set dst (CmpU3 src1 src2));
16352 effect(KILL flags);
16353
16354 ins_cost(275); // XXX
16355 format %{ "cmpl $src1, $src2\t# CmpL3\n\t"
16356 "movl $dst, -1\n\t"
16357 "jb,u done\n\t"
16358 "setcc $dst \t# emits setne + movzbl or setzune for APX"
16359 "done:" %}
16360 ins_encode %{
16361 Label done;
16362 __ cmpl($src1$$Register, $src2$$Register);
16363 __ movl($dst$$Register, -1);
16364 __ jccb(Assembler::below, done);
16365 __ setcc(Assembler::notZero, $dst$$Register);
16366 __ bind(done);
16367 %}
16368 ins_pipe(pipe_slow);
16369 %}
16370
16371 // Manifest a CmpL result in an integer register. Very painful.
16372 // This is the test to avoid.
16373 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
16374 %{
16375 match(Set dst (CmpL3 src1 src2));
16376 effect(KILL flags);
16377
16378 ins_cost(275); // XXX
16379 format %{ "cmpq $src1, $src2\t# CmpL3\n\t"
16380 "movl $dst, -1\n\t"
16381 "jl,s done\n\t"
16382 "setcc $dst \t# emits setne + movzbl or setzune for APX"
16383 "done:" %}
16384 ins_encode %{
16385 Label done;
16386 __ cmpq($src1$$Register, $src2$$Register);
16387 __ movl($dst$$Register, -1);
16388 __ jccb(Assembler::less, done);
16389 __ setcc(Assembler::notZero, $dst$$Register);
16390 __ bind(done);
16391 %}
16392 ins_pipe(pipe_slow);
16393 %}
16394
16395 // Manifest a CmpUL result in an integer register. Very painful.
16396 // This is the test to avoid.
16397 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
16398 %{
16399 match(Set dst (CmpUL3 src1 src2));
16400 effect(KILL flags);
16401
16402 ins_cost(275); // XXX
16403 format %{ "cmpq $src1, $src2\t# CmpL3\n\t"
16404 "movl $dst, -1\n\t"
16405 "jb,u done\n\t"
16406 "setcc $dst \t# emits setne + movzbl or setzune for APX"
16407 "done:" %}
16408 ins_encode %{
16409 Label done;
16410 __ cmpq($src1$$Register, $src2$$Register);
16411 __ movl($dst$$Register, -1);
16412 __ jccb(Assembler::below, done);
16413 __ setcc(Assembler::notZero, $dst$$Register);
16414 __ bind(done);
16415 %}
16416 ins_pipe(pipe_slow);
16417 %}
16418
16419 // Unsigned long compare Instructions; really, same as signed long except they
16420 // produce an rFlagsRegU instead of rFlagsReg.
16421 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
16422 %{
16423 match(Set cr (CmpUL op1 op2));
16424
16425 format %{ "cmpq $op1, $op2\t# unsigned" %}
16426 ins_encode %{
16427 __ cmpq($op1$$Register, $op2$$Register);
16428 %}
16429 ins_pipe(ialu_cr_reg_reg);
16430 %}
16431
16432 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
16433 %{
16434 match(Set cr (CmpUL op1 op2));
16435
16436 format %{ "cmpq $op1, $op2\t# unsigned" %}
16437 ins_encode %{
16438 __ cmpq($op1$$Register, $op2$$constant);
16439 %}
16440 ins_pipe(ialu_cr_reg_imm);
16441 %}
16442
16443 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
16444 %{
16445 match(Set cr (CmpUL op1 (LoadL op2)));
16446
16447 format %{ "cmpq $op1, $op2\t# unsigned" %}
16448 ins_encode %{
16449 __ cmpq($op1$$Register, $op2$$Address);
16450 %}
16451 ins_pipe(ialu_cr_reg_mem);
16452 %}
16453
16454 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
16455 %{
16456 match(Set cr (CmpUL src zero));
16457
16458 format %{ "testq $src, $src\t# unsigned" %}
16459 ins_encode %{
16460 __ testq($src$$Register, $src$$Register);
16461 %}
16462 ins_pipe(ialu_cr_reg_imm);
16463 %}
16464
16465 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
16466 %{
16467 match(Set cr (CmpI (LoadB mem) imm));
16468
16469 ins_cost(125);
16470 format %{ "cmpb $mem, $imm" %}
16471 ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
16472 ins_pipe(ialu_cr_reg_mem);
16473 %}
16474
16475 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
16476 %{
16477 match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
16478
16479 ins_cost(125);
16480 format %{ "testb $mem, $imm\t# ubyte" %}
16481 ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
16482 ins_pipe(ialu_cr_reg_mem);
16483 %}
16484
16485 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
16486 %{
16487 match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
16488
16489 ins_cost(125);
16490 format %{ "testb $mem, $imm\t# byte" %}
16491 ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
16492 ins_pipe(ialu_cr_reg_mem);
16493 %}
16494
16495 //----------Max and Min--------------------------------------------------------
16496 // Min Instructions
16497
16498 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
16499 %{
16500 predicate(!UseAPX);
16501 effect(USE_DEF dst, USE src, USE cr);
16502
16503 format %{ "cmovlgt $dst, $src\t# min" %}
16504 ins_encode %{
16505 __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
16506 %}
16507 ins_pipe(pipe_cmov_reg);
16508 %}
16509
16510 instruct cmovI_reg_g_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
16511 %{
16512 predicate(UseAPX);
16513 effect(DEF dst, USE src1, USE src2, USE cr);
16514
16515 format %{ "ecmovlgt $dst, $src1, $src2\t# min ndd" %}
16516 ins_encode %{
16517 __ ecmovl(Assembler::greater, $dst$$Register, $src1$$Register, $src2$$Register);
16518 %}
16519 ins_pipe(pipe_cmov_reg);
16520 %}
16521
16522 instruct minI_rReg(rRegI dst, rRegI src)
16523 %{
16524 predicate(!UseAPX);
16525 match(Set dst (MinI dst src));
16526
16527 ins_cost(200);
16528 expand %{
16529 rFlagsReg cr;
16530 compI_rReg(cr, dst, src);
16531 cmovI_reg_g(dst, src, cr);
16532 %}
16533 %}
16534
16535 instruct minI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
16536 %{
16537 predicate(UseAPX);
16538 match(Set dst (MinI src1 src2));
16539 effect(DEF dst, USE src1, USE src2);
16540 flag(PD::Flag_ndd_demotable_opr1);
16541
16542 ins_cost(200);
16543 expand %{
16544 rFlagsReg cr;
16545 compI_rReg(cr, src1, src2);
16546 cmovI_reg_g_ndd(dst, src1, src2, cr);
16547 %}
16548 %}
16549
16550 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
16551 %{
16552 predicate(!UseAPX);
16553 effect(USE_DEF dst, USE src, USE cr);
16554
16555 format %{ "cmovllt $dst, $src\t# max" %}
16556 ins_encode %{
16557 __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
16558 %}
16559 ins_pipe(pipe_cmov_reg);
16560 %}
16561
16562 instruct cmovI_reg_l_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
16563 %{
16564 predicate(UseAPX);
16565 effect(DEF dst, USE src1, USE src2, USE cr);
16566
16567 format %{ "ecmovllt $dst, $src1, $src2\t# max ndd" %}
16568 ins_encode %{
16569 __ ecmovl(Assembler::less, $dst$$Register, $src1$$Register, $src2$$Register);
16570 %}
16571 ins_pipe(pipe_cmov_reg);
16572 %}
16573
16574 instruct maxI_rReg(rRegI dst, rRegI src)
16575 %{
16576 predicate(!UseAPX);
16577 match(Set dst (MaxI dst src));
16578
16579 ins_cost(200);
16580 expand %{
16581 rFlagsReg cr;
16582 compI_rReg(cr, dst, src);
16583 cmovI_reg_l(dst, src, cr);
16584 %}
16585 %}
16586
16587 instruct maxI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
16588 %{
16589 predicate(UseAPX);
16590 match(Set dst (MaxI src1 src2));
16591 effect(DEF dst, USE src1, USE src2);
16592 flag(PD::Flag_ndd_demotable_opr1);
16593
16594 ins_cost(200);
16595 expand %{
16596 rFlagsReg cr;
16597 compI_rReg(cr, src1, src2);
16598 cmovI_reg_l_ndd(dst, src1, src2, cr);
16599 %}
16600 %}
16601
16602 // ============================================================================
16603 // Branch Instructions
16604
16605 // Jump Direct - Label defines a relative address from JMP+1
16606 instruct jmpDir(label labl)
16607 %{
16608 match(Goto);
16609 effect(USE labl);
16610
16611 ins_cost(300);
16612 format %{ "jmp $labl" %}
16613 size(5);
16614 ins_encode %{
16615 Label* L = $labl$$label;
16616 __ jmp(*L, false); // Always long jump
16617 %}
16618 ins_pipe(pipe_jmp);
16619 %}
16620
16621 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16622 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
16623 %{
16624 match(If cop cr);
16625 effect(USE labl);
16626
16627 ins_cost(300);
16628 format %{ "j$cop $labl" %}
16629 size(6);
16630 ins_encode %{
16631 Label* L = $labl$$label;
16632 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16633 %}
16634 ins_pipe(pipe_jcc);
16635 %}
16636
16637 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16638 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
16639 %{
16640 match(CountedLoopEnd cop cr);
16641 effect(USE labl);
16642
16643 ins_cost(300);
16644 format %{ "j$cop $labl\t# loop end" %}
16645 size(6);
16646 ins_encode %{
16647 Label* L = $labl$$label;
16648 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16649 %}
16650 ins_pipe(pipe_jcc);
16651 %}
16652
16653 // Jump Direct Conditional - using unsigned comparison
16654 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
16655 match(If cop cmp);
16656 effect(USE labl);
16657
16658 ins_cost(300);
16659 format %{ "j$cop,u $labl" %}
16660 size(6);
16661 ins_encode %{
16662 Label* L = $labl$$label;
16663 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16664 %}
16665 ins_pipe(pipe_jcc);
16666 %}
16667
16668 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
16669 match(If cop cmp);
16670 effect(USE labl);
16671
16672 ins_cost(200);
16673 format %{ "j$cop,u $labl" %}
16674 size(6);
16675 ins_encode %{
16676 Label* L = $labl$$label;
16677 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16678 %}
16679 ins_pipe(pipe_jcc);
16680 %}
16681
16682 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
16683 match(If cop cmp);
16684 effect(USE labl);
16685
16686 ins_cost(200);
16687 format %{ $$template
16688 if ($cop$$cmpcode == Assembler::notEqual) {
16689 $$emit$$"jp,u $labl\n\t"
16690 $$emit$$"j$cop,u $labl"
16691 } else {
16692 $$emit$$"jp,u done\n\t"
16693 $$emit$$"j$cop,u $labl\n\t"
16694 $$emit$$"done:"
16695 }
16696 %}
16697 ins_encode %{
16698 Label* l = $labl$$label;
16699 if ($cop$$cmpcode == Assembler::notEqual) {
16700 __ jcc(Assembler::parity, *l, false);
16701 __ jcc(Assembler::notEqual, *l, false);
16702 } else if ($cop$$cmpcode == Assembler::equal) {
16703 Label done;
16704 __ jccb(Assembler::parity, done);
16705 __ jcc(Assembler::equal, *l, false);
16706 __ bind(done);
16707 } else {
16708 ShouldNotReachHere();
16709 }
16710 %}
16711 ins_pipe(pipe_jcc);
16712 %}
16713
16714 // Jump Direct Conditional - using signed and unsigned comparison
16715 instruct jmpConUCFE(cmpOpUCFE cop, rFlagsRegUCFE cmp, label labl) %{
16716 match(If cop cmp);
16717 effect(USE labl);
16718
16719 ins_cost(200);
16720 format %{ "j$cop,su $labl" %}
16721 size(6);
16722 ins_encode %{
16723 Label* L = $labl$$label;
16724 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16725 %}
16726 ins_pipe(pipe_jcc);
16727 %}
16728
16729 // ============================================================================
16730 // The 2nd slow-half of a subtype check. Scan the subklass's 2ndary
16731 // superklass array for an instance of the superklass. Set a hidden
16732 // internal cache on a hit (cache is checked with exposed code in
16733 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16734 // encoding ALSO sets flags.
16735
16736 instruct partialSubtypeCheck(rdi_RegP result,
16737 rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
16738 rFlagsReg cr)
16739 %{
16740 match(Set result (PartialSubtypeCheck sub super));
16741 predicate(!UseSecondarySupersTable);
16742 effect(KILL rcx, KILL cr);
16743
16744 ins_cost(1100); // slightly larger than the next version
16745 format %{ "movq rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
16746 "movl rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
16747 "addq rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
16748 "repne scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
16749 "jne,s miss\t\t# Missed: rdi not-zero\n\t"
16750 "movq [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
16751 "xorq $result, $result\t\t Hit: rdi zero\n\t"
16752 "miss:\t" %}
16753
16754 ins_encode %{
16755 Label miss;
16756 // NB: Callers may assume that, when $result is a valid register,
16757 // check_klass_subtype_slow_path_linear sets it to a nonzero
16758 // value.
16759 __ check_klass_subtype_slow_path_linear($sub$$Register, $super$$Register,
16760 $rcx$$Register, $result$$Register,
16761 nullptr, &miss,
16762 /*set_cond_codes:*/ true);
16763 __ xorptr($result$$Register, $result$$Register);
16764 __ bind(miss);
16765 %}
16766
16767 ins_pipe(pipe_slow);
16768 %}
16769
16770 // ============================================================================
16771 // Two versions of hashtable-based partialSubtypeCheck, both used when
16772 // we need to search for a super class in the secondary supers array.
16773 // The first is used when we don't know _a priori_ the class being
16774 // searched for. The second, far more common, is used when we do know:
16775 // this is used for instanceof, checkcast, and any case where C2 can
16776 // determine it by constant propagation.
16777
16778 instruct partialSubtypeCheckVarSuper(rsi_RegP sub, rax_RegP super, rdi_RegP result,
16779 rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
16780 rFlagsReg cr)
16781 %{
16782 match(Set result (PartialSubtypeCheck sub super));
16783 predicate(UseSecondarySupersTable);
16784 effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
16785
16786 ins_cost(1000);
16787 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16788
16789 ins_encode %{
16790 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register, $temp1$$Register, $temp2$$Register,
16791 $temp3$$Register, $temp4$$Register, $result$$Register);
16792 %}
16793
16794 ins_pipe(pipe_slow);
16795 %}
16796
16797 instruct partialSubtypeCheckConstSuper(rsi_RegP sub, rax_RegP super_reg, immP super_con, rdi_RegP result,
16798 rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
16799 rFlagsReg cr)
16800 %{
16801 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16802 predicate(UseSecondarySupersTable);
16803 effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
16804
16805 ins_cost(700); // smaller than the next version
16806 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16807
16808 ins_encode %{
16809 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16810 if (InlineSecondarySupersTest) {
16811 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register, $temp1$$Register, $temp2$$Register,
16812 $temp3$$Register, $temp4$$Register, $result$$Register,
16813 super_klass_slot);
16814 } else {
16815 __ call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16816 }
16817 %}
16818
16819 ins_pipe(pipe_slow);
16820 %}
16821
16822 // ============================================================================
16823 // Branch Instructions -- short offset versions
16824 //
16825 // These instructions are used to replace jumps of a long offset (the default
16826 // match) with jumps of a shorter offset. These instructions are all tagged
16827 // with the ins_short_branch attribute, which causes the ADLC to suppress the
16828 // match rules in general matching. Instead, the ADLC generates a conversion
16829 // method in the MachNode which can be used to do in-place replacement of the
16830 // long variant with the shorter variant. The compiler will determine if a
16831 // branch can be taken by the is_short_branch_offset() predicate in the machine
16832 // specific code section of the file.
16833
16834 // Jump Direct - Label defines a relative address from JMP+1
16835 instruct jmpDir_short(label labl) %{
16836 match(Goto);
16837 effect(USE labl);
16838
16839 ins_cost(300);
16840 format %{ "jmp,s $labl" %}
16841 size(2);
16842 ins_encode %{
16843 Label* L = $labl$$label;
16844 __ jmpb(*L);
16845 %}
16846 ins_pipe(pipe_jmp);
16847 ins_short_branch(1);
16848 %}
16849
16850 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16851 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
16852 match(If cop cr);
16853 effect(USE labl);
16854
16855 ins_cost(300);
16856 format %{ "j$cop,s $labl" %}
16857 size(2);
16858 ins_encode %{
16859 Label* L = $labl$$label;
16860 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16861 %}
16862 ins_pipe(pipe_jcc);
16863 ins_short_branch(1);
16864 %}
16865
16866 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16867 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
16868 match(CountedLoopEnd cop cr);
16869 effect(USE labl);
16870
16871 ins_cost(300);
16872 format %{ "j$cop,s $labl\t# loop end" %}
16873 size(2);
16874 ins_encode %{
16875 Label* L = $labl$$label;
16876 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16877 %}
16878 ins_pipe(pipe_jcc);
16879 ins_short_branch(1);
16880 %}
16881
16882 // Jump Direct Conditional - using unsigned comparison
16883 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
16884 match(If cop cmp);
16885 effect(USE labl);
16886
16887 ins_cost(300);
16888 format %{ "j$cop,us $labl" %}
16889 size(2);
16890 ins_encode %{
16891 Label* L = $labl$$label;
16892 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16893 %}
16894 ins_pipe(pipe_jcc);
16895 ins_short_branch(1);
16896 %}
16897
16898 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
16899 match(If cop cmp);
16900 effect(USE labl);
16901
16902 ins_cost(300);
16903 format %{ "j$cop,us $labl" %}
16904 size(2);
16905 ins_encode %{
16906 Label* L = $labl$$label;
16907 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16908 %}
16909 ins_pipe(pipe_jcc);
16910 ins_short_branch(1);
16911 %}
16912
16913 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
16914 match(If cop cmp);
16915 effect(USE labl);
16916
16917 ins_cost(300);
16918 format %{ $$template
16919 if ($cop$$cmpcode == Assembler::notEqual) {
16920 $$emit$$"jp,u,s $labl\n\t"
16921 $$emit$$"j$cop,u,s $labl"
16922 } else {
16923 $$emit$$"jp,u,s done\n\t"
16924 $$emit$$"j$cop,u,s $labl\n\t"
16925 $$emit$$"done:"
16926 }
16927 %}
16928 size(4);
16929 ins_encode %{
16930 Label* l = $labl$$label;
16931 if ($cop$$cmpcode == Assembler::notEqual) {
16932 __ jccb(Assembler::parity, *l);
16933 __ jccb(Assembler::notEqual, *l);
16934 } else if ($cop$$cmpcode == Assembler::equal) {
16935 Label done;
16936 __ jccb(Assembler::parity, done);
16937 __ jccb(Assembler::equal, *l);
16938 __ bind(done);
16939 } else {
16940 ShouldNotReachHere();
16941 }
16942 %}
16943 ins_pipe(pipe_jcc);
16944 ins_short_branch(1);
16945 %}
16946
16947 // Jump Direct Conditional - using signed and unsigned comparison
16948 instruct jmpConUCFE_short(cmpOpUCFE cop, rFlagsRegUCFE cmp, label labl) %{
16949 match(If cop cmp);
16950 effect(USE labl);
16951
16952 ins_cost(300);
16953 format %{ "j$cop,sus $labl" %}
16954 size(2);
16955 ins_encode %{
16956 Label* L = $labl$$label;
16957 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16958 %}
16959 ins_pipe(pipe_jcc);
16960 ins_short_branch(1);
16961 %}
16962
16963 // ============================================================================
16964 // inlined locking and unlocking
16965
16966 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
16967 match(Set cr (FastLock object box));
16968 effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
16969 ins_cost(300);
16970 format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
16971 ins_encode %{
16972 __ fast_lock($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
16973 %}
16974 ins_pipe(pipe_slow);
16975 %}
16976
16977 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
16978 match(Set cr (FastUnlock object rax_reg));
16979 effect(TEMP tmp, USE_KILL rax_reg);
16980 ins_cost(300);
16981 format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
16982 ins_encode %{
16983 __ fast_unlock($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
16984 %}
16985 ins_pipe(pipe_slow);
16986 %}
16987
16988
16989 // ============================================================================
16990 // Safepoint Instructions
16991 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
16992 %{
16993 match(SafePoint poll);
16994 effect(KILL cr, USE poll);
16995
16996 format %{ "testl rax, [$poll]\t"
16997 "# Safepoint: poll for GC" %}
16998 ins_cost(125);
16999 ins_encode %{
17000 __ relocate(relocInfo::poll_type);
17001 address pre_pc = __ pc();
17002 __ testl(rax, Address($poll$$Register, 0));
17003 assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
17004 %}
17005 ins_pipe(ialu_reg_mem);
17006 %}
17007
17008 instruct mask_all_evexL(kReg dst, rRegL src) %{
17009 match(Set dst (MaskAll src));
17010 format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
17011 ins_encode %{
17012 int mask_len = Matcher::vector_length(this);
17013 __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
17014 %}
17015 ins_pipe( pipe_slow );
17016 %}
17017
17018 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
17019 predicate(Matcher::vector_length(n) > 32);
17020 match(Set dst (MaskAll src));
17021 effect(TEMP tmp);
17022 format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
17023 ins_encode %{
17024 int mask_len = Matcher::vector_length(this);
17025 __ movslq($tmp$$Register, $src$$Register);
17026 __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
17027 %}
17028 ins_pipe( pipe_slow );
17029 %}
17030
17031 // ============================================================================
17032 // Procedure Call/Return Instructions
17033 // Call Java Static Instruction
17034 // Note: If this code changes, the corresponding ret_addr_offset() and
17035 // compute_padding() functions will have to be adjusted.
17036 instruct CallStaticJavaDirect(method meth) %{
17037 match(CallStaticJava);
17038 effect(USE meth);
17039
17040 ins_cost(300);
17041 format %{ "call,static " %}
17042 opcode(0xE8); /* E8 cd */
17043 ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
17044 ins_pipe(pipe_slow);
17045 ins_alignment(4);
17046 %}
17047
17048 // Call Java Dynamic Instruction
17049 // Note: If this code changes, the corresponding ret_addr_offset() and
17050 // compute_padding() functions will have to be adjusted.
17051 instruct CallDynamicJavaDirect(method meth)
17052 %{
17053 match(CallDynamicJava);
17054 effect(USE meth);
17055
17056 ins_cost(300);
17057 format %{ "movq rax, #Universe::non_oop_word()\n\t"
17058 "call,dynamic " %}
17059 ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
17060 ins_pipe(pipe_slow);
17061 ins_alignment(4);
17062 %}
17063
17064 // Call Runtime Instruction
17065 instruct CallRuntimeDirect(method meth)
17066 %{
17067 match(CallRuntime);
17068 effect(USE meth);
17069
17070 ins_cost(300);
17071 format %{ "call,runtime " %}
17072 ins_encode(clear_avx, Java_To_Runtime(meth));
17073 ins_pipe(pipe_slow);
17074 %}
17075
17076 // Call runtime without safepoint
17077 instruct CallLeafDirect(method meth)
17078 %{
17079 match(CallLeaf);
17080 effect(USE meth);
17081
17082 ins_cost(300);
17083 format %{ "call_leaf,runtime " %}
17084 ins_encode(clear_avx, Java_To_Runtime(meth));
17085 ins_pipe(pipe_slow);
17086 %}
17087
17088 // Call runtime without safepoint and with vector arguments
17089 instruct CallLeafDirectVector(method meth)
17090 %{
17091 match(CallLeafVector);
17092 effect(USE meth);
17093
17094 ins_cost(300);
17095 format %{ "call_leaf,vector " %}
17096 ins_encode(Java_To_Runtime(meth));
17097 ins_pipe(pipe_slow);
17098 %}
17099
17100 // Call runtime without safepoint
17101 // entry point is null, target holds the address to call
17102 instruct CallLeafNoFPInDirect(rRegP target)
17103 %{
17104 predicate(n->as_Call()->entry_point() == nullptr);
17105 match(CallLeafNoFP target);
17106
17107 ins_cost(300);
17108 format %{ "call_leaf_nofp,runtime indirect " %}
17109 ins_encode %{
17110 __ call($target$$Register);
17111 %}
17112
17113 ins_pipe(pipe_slow);
17114 %}
17115
17116 // Call runtime without safepoint
17117 instruct CallLeafNoFPDirect(method meth)
17118 %{
17119 predicate(n->as_Call()->entry_point() != nullptr);
17120 match(CallLeafNoFP);
17121 effect(USE meth);
17122
17123 ins_cost(300);
17124 format %{ "call_leaf_nofp,runtime " %}
17125 ins_encode(clear_avx, Java_To_Runtime(meth));
17126 ins_pipe(pipe_slow);
17127 %}
17128
17129 // Return Instruction
17130 // Remove the return address & jump to it.
17131 // Notice: We always emit a nop after a ret to make sure there is room
17132 // for safepoint patching
17133 instruct Ret()
17134 %{
17135 match(Return);
17136
17137 format %{ "ret" %}
17138 ins_encode %{
17139 __ ret(0);
17140 %}
17141 ins_pipe(pipe_jmp);
17142 %}
17143
17144 // Tail Call; Jump from runtime stub to Java code.
17145 // Also known as an 'interprocedural jump'.
17146 // Target of jump will eventually return to caller.
17147 // TailJump below removes the return address.
17148 // Don't use rbp for 'jump_target' because a MachEpilogNode has already been
17149 // emitted just above the TailCall which has reset rbp to the caller state.
17150 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
17151 %{
17152 match(TailCall jump_target method_ptr);
17153
17154 ins_cost(300);
17155 format %{ "jmp $jump_target\t# rbx holds method" %}
17156 ins_encode %{
17157 __ jmp($jump_target$$Register);
17158 %}
17159 ins_pipe(pipe_jmp);
17160 %}
17161
17162 // Tail Jump; remove the return address; jump to target.
17163 // TailCall above leaves the return address around.
17164 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
17165 %{
17166 match(TailJump jump_target ex_oop);
17167
17168 ins_cost(300);
17169 format %{ "popq rdx\t# pop return address\n\t"
17170 "jmp $jump_target" %}
17171 ins_encode %{
17172 __ popq(as_Register(RDX_enc));
17173 __ jmp($jump_target$$Register);
17174 %}
17175 ins_pipe(pipe_jmp);
17176 %}
17177
17178 // Forward exception.
17179 instruct ForwardExceptionjmp()
17180 %{
17181 match(ForwardException);
17182
17183 format %{ "jmp forward_exception_stub" %}
17184 ins_encode %{
17185 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()), noreg);
17186 %}
17187 ins_pipe(pipe_jmp);
17188 %}
17189
17190 // Create exception oop: created by stack-crawling runtime code.
17191 // Created exception is now available to this handler, and is setup
17192 // just prior to jumping to this handler. No code emitted.
17193 instruct CreateException(rax_RegP ex_oop)
17194 %{
17195 match(Set ex_oop (CreateEx));
17196
17197 size(0);
17198 // use the following format syntax
17199 format %{ "# exception oop is in rax; no code emitted" %}
17200 ins_encode();
17201 ins_pipe(empty);
17202 %}
17203
17204 // Rethrow exception:
17205 // The exception oop will come in the first argument position.
17206 // Then JUMP (not call) to the rethrow stub code.
17207 instruct RethrowException()
17208 %{
17209 match(Rethrow);
17210
17211 // use the following format syntax
17212 format %{ "jmp rethrow_stub" %}
17213 ins_encode %{
17214 __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
17215 %}
17216 ins_pipe(pipe_jmp);
17217 %}
17218
17219 // ============================================================================
17220 // This name is KNOWN by the ADLC and cannot be changed.
17221 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17222 // for this guy.
17223 instruct tlsLoadP(r15_RegP dst) %{
17224 match(Set dst (ThreadLocal));
17225 effect(DEF dst);
17226
17227 size(0);
17228 format %{ "# TLS is in R15" %}
17229 ins_encode( /*empty encoding*/ );
17230 ins_pipe(ialu_reg_reg);
17231 %}
17232
17233 instruct addF_reg(regF dst, regF src) %{
17234 predicate(UseAVX == 0);
17235 match(Set dst (AddF dst src));
17236
17237 format %{ "addss $dst, $src" %}
17238 ins_cost(150);
17239 ins_encode %{
17240 __ addss($dst$$XMMRegister, $src$$XMMRegister);
17241 %}
17242 ins_pipe(pipe_slow);
17243 %}
17244
17245 instruct addF_mem(regF dst, memory src) %{
17246 predicate(UseAVX == 0);
17247 match(Set dst (AddF dst (LoadF src)));
17248
17249 format %{ "addss $dst, $src" %}
17250 ins_cost(150);
17251 ins_encode %{
17252 __ addss($dst$$XMMRegister, $src$$Address);
17253 %}
17254 ins_pipe(pipe_slow);
17255 %}
17256
17257 instruct addF_imm(regF dst, immF con) %{
17258 predicate(UseAVX == 0);
17259 match(Set dst (AddF dst con));
17260 format %{ "addss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
17261 ins_cost(150);
17262 ins_encode %{
17263 __ addss($dst$$XMMRegister, $constantaddress($con));
17264 %}
17265 ins_pipe(pipe_slow);
17266 %}
17267
17268 instruct addF_reg_reg(regF dst, regF src1, regF src2) %{
17269 predicate(UseAVX > 0);
17270 match(Set dst (AddF src1 src2));
17271
17272 format %{ "vaddss $dst, $src1, $src2" %}
17273 ins_cost(150);
17274 ins_encode %{
17275 __ vaddss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17276 %}
17277 ins_pipe(pipe_slow);
17278 %}
17279
17280 instruct addF_reg_mem(regF dst, regF src1, memory src2) %{
17281 predicate(UseAVX > 0);
17282 match(Set dst (AddF src1 (LoadF src2)));
17283
17284 format %{ "vaddss $dst, $src1, $src2" %}
17285 ins_cost(150);
17286 ins_encode %{
17287 __ vaddss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17288 %}
17289 ins_pipe(pipe_slow);
17290 %}
17291
17292 instruct addF_reg_imm(regF dst, regF src, immF con) %{
17293 predicate(UseAVX > 0);
17294 match(Set dst (AddF src con));
17295
17296 format %{ "vaddss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
17297 ins_cost(150);
17298 ins_encode %{
17299 __ vaddss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17300 %}
17301 ins_pipe(pipe_slow);
17302 %}
17303
17304 instruct addD_reg(regD dst, regD src) %{
17305 predicate(UseAVX == 0);
17306 match(Set dst (AddD dst src));
17307
17308 format %{ "addsd $dst, $src" %}
17309 ins_cost(150);
17310 ins_encode %{
17311 __ addsd($dst$$XMMRegister, $src$$XMMRegister);
17312 %}
17313 ins_pipe(pipe_slow);
17314 %}
17315
17316 instruct addD_mem(regD dst, memory src) %{
17317 predicate(UseAVX == 0);
17318 match(Set dst (AddD dst (LoadD src)));
17319
17320 format %{ "addsd $dst, $src" %}
17321 ins_cost(150);
17322 ins_encode %{
17323 __ addsd($dst$$XMMRegister, $src$$Address);
17324 %}
17325 ins_pipe(pipe_slow);
17326 %}
17327
17328 instruct addD_imm(regD dst, immD con) %{
17329 predicate(UseAVX == 0);
17330 match(Set dst (AddD dst con));
17331 format %{ "addsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17332 ins_cost(150);
17333 ins_encode %{
17334 __ addsd($dst$$XMMRegister, $constantaddress($con));
17335 %}
17336 ins_pipe(pipe_slow);
17337 %}
17338
17339 instruct addD_reg_reg(regD dst, regD src1, regD src2) %{
17340 predicate(UseAVX > 0);
17341 match(Set dst (AddD src1 src2));
17342
17343 format %{ "vaddsd $dst, $src1, $src2" %}
17344 ins_cost(150);
17345 ins_encode %{
17346 __ vaddsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17347 %}
17348 ins_pipe(pipe_slow);
17349 %}
17350
17351 instruct addD_reg_mem(regD dst, regD src1, memory src2) %{
17352 predicate(UseAVX > 0);
17353 match(Set dst (AddD src1 (LoadD src2)));
17354
17355 format %{ "vaddsd $dst, $src1, $src2" %}
17356 ins_cost(150);
17357 ins_encode %{
17358 __ vaddsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17359 %}
17360 ins_pipe(pipe_slow);
17361 %}
17362
17363 instruct addD_reg_imm(regD dst, regD src, immD con) %{
17364 predicate(UseAVX > 0);
17365 match(Set dst (AddD src con));
17366
17367 format %{ "vaddsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17368 ins_cost(150);
17369 ins_encode %{
17370 __ vaddsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17371 %}
17372 ins_pipe(pipe_slow);
17373 %}
17374
17375 instruct subF_reg(regF dst, regF src) %{
17376 predicate(UseAVX == 0);
17377 match(Set dst (SubF dst src));
17378
17379 format %{ "subss $dst, $src" %}
17380 ins_cost(150);
17381 ins_encode %{
17382 __ subss($dst$$XMMRegister, $src$$XMMRegister);
17383 %}
17384 ins_pipe(pipe_slow);
17385 %}
17386
17387 instruct subF_mem(regF dst, memory src) %{
17388 predicate(UseAVX == 0);
17389 match(Set dst (SubF dst (LoadF src)));
17390
17391 format %{ "subss $dst, $src" %}
17392 ins_cost(150);
17393 ins_encode %{
17394 __ subss($dst$$XMMRegister, $src$$Address);
17395 %}
17396 ins_pipe(pipe_slow);
17397 %}
17398
17399 instruct subF_imm(regF dst, immF con) %{
17400 predicate(UseAVX == 0);
17401 match(Set dst (SubF dst con));
17402 format %{ "subss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
17403 ins_cost(150);
17404 ins_encode %{
17405 __ subss($dst$$XMMRegister, $constantaddress($con));
17406 %}
17407 ins_pipe(pipe_slow);
17408 %}
17409
17410 instruct subF_reg_reg(regF dst, regF src1, regF src2) %{
17411 predicate(UseAVX > 0);
17412 match(Set dst (SubF src1 src2));
17413
17414 format %{ "vsubss $dst, $src1, $src2" %}
17415 ins_cost(150);
17416 ins_encode %{
17417 __ vsubss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17418 %}
17419 ins_pipe(pipe_slow);
17420 %}
17421
17422 instruct subF_reg_mem(regF dst, regF src1, memory src2) %{
17423 predicate(UseAVX > 0);
17424 match(Set dst (SubF src1 (LoadF src2)));
17425
17426 format %{ "vsubss $dst, $src1, $src2" %}
17427 ins_cost(150);
17428 ins_encode %{
17429 __ vsubss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17430 %}
17431 ins_pipe(pipe_slow);
17432 %}
17433
17434 instruct subF_reg_imm(regF dst, regF src, immF con) %{
17435 predicate(UseAVX > 0);
17436 match(Set dst (SubF src con));
17437
17438 format %{ "vsubss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
17439 ins_cost(150);
17440 ins_encode %{
17441 __ vsubss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17442 %}
17443 ins_pipe(pipe_slow);
17444 %}
17445
17446 instruct subD_reg(regD dst, regD src) %{
17447 predicate(UseAVX == 0);
17448 match(Set dst (SubD dst src));
17449
17450 format %{ "subsd $dst, $src" %}
17451 ins_cost(150);
17452 ins_encode %{
17453 __ subsd($dst$$XMMRegister, $src$$XMMRegister);
17454 %}
17455 ins_pipe(pipe_slow);
17456 %}
17457
17458 instruct subD_mem(regD dst, memory src) %{
17459 predicate(UseAVX == 0);
17460 match(Set dst (SubD dst (LoadD src)));
17461
17462 format %{ "subsd $dst, $src" %}
17463 ins_cost(150);
17464 ins_encode %{
17465 __ subsd($dst$$XMMRegister, $src$$Address);
17466 %}
17467 ins_pipe(pipe_slow);
17468 %}
17469
17470 instruct subD_imm(regD dst, immD con) %{
17471 predicate(UseAVX == 0);
17472 match(Set dst (SubD dst con));
17473 format %{ "subsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17474 ins_cost(150);
17475 ins_encode %{
17476 __ subsd($dst$$XMMRegister, $constantaddress($con));
17477 %}
17478 ins_pipe(pipe_slow);
17479 %}
17480
17481 instruct subD_reg_reg(regD dst, regD src1, regD src2) %{
17482 predicate(UseAVX > 0);
17483 match(Set dst (SubD src1 src2));
17484
17485 format %{ "vsubsd $dst, $src1, $src2" %}
17486 ins_cost(150);
17487 ins_encode %{
17488 __ vsubsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17489 %}
17490 ins_pipe(pipe_slow);
17491 %}
17492
17493 instruct subD_reg_mem(regD dst, regD src1, memory src2) %{
17494 predicate(UseAVX > 0);
17495 match(Set dst (SubD src1 (LoadD src2)));
17496
17497 format %{ "vsubsd $dst, $src1, $src2" %}
17498 ins_cost(150);
17499 ins_encode %{
17500 __ vsubsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17501 %}
17502 ins_pipe(pipe_slow);
17503 %}
17504
17505 instruct subD_reg_imm(regD dst, regD src, immD con) %{
17506 predicate(UseAVX > 0);
17507 match(Set dst (SubD src con));
17508
17509 format %{ "vsubsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17510 ins_cost(150);
17511 ins_encode %{
17512 __ vsubsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17513 %}
17514 ins_pipe(pipe_slow);
17515 %}
17516
17517 instruct mulF_reg(regF dst, regF src) %{
17518 predicate(UseAVX == 0);
17519 match(Set dst (MulF dst src));
17520
17521 format %{ "mulss $dst, $src" %}
17522 ins_cost(150);
17523 ins_encode %{
17524 __ mulss($dst$$XMMRegister, $src$$XMMRegister);
17525 %}
17526 ins_pipe(pipe_slow);
17527 %}
17528
17529 instruct mulF_mem(regF dst, memory src) %{
17530 predicate(UseAVX == 0);
17531 match(Set dst (MulF dst (LoadF src)));
17532
17533 format %{ "mulss $dst, $src" %}
17534 ins_cost(150);
17535 ins_encode %{
17536 __ mulss($dst$$XMMRegister, $src$$Address);
17537 %}
17538 ins_pipe(pipe_slow);
17539 %}
17540
17541 instruct mulF_imm(regF dst, immF con) %{
17542 predicate(UseAVX == 0);
17543 match(Set dst (MulF dst con));
17544 format %{ "mulss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
17545 ins_cost(150);
17546 ins_encode %{
17547 __ mulss($dst$$XMMRegister, $constantaddress($con));
17548 %}
17549 ins_pipe(pipe_slow);
17550 %}
17551
17552 instruct mulF_reg_reg(regF dst, regF src1, regF src2) %{
17553 predicate(UseAVX > 0);
17554 match(Set dst (MulF src1 src2));
17555
17556 format %{ "vmulss $dst, $src1, $src2" %}
17557 ins_cost(150);
17558 ins_encode %{
17559 __ vmulss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17560 %}
17561 ins_pipe(pipe_slow);
17562 %}
17563
17564 instruct mulF_reg_mem(regF dst, regF src1, memory src2) %{
17565 predicate(UseAVX > 0);
17566 match(Set dst (MulF src1 (LoadF src2)));
17567
17568 format %{ "vmulss $dst, $src1, $src2" %}
17569 ins_cost(150);
17570 ins_encode %{
17571 __ vmulss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17572 %}
17573 ins_pipe(pipe_slow);
17574 %}
17575
17576 instruct mulF_reg_imm(regF dst, regF src, immF con) %{
17577 predicate(UseAVX > 0);
17578 match(Set dst (MulF src con));
17579
17580 format %{ "vmulss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
17581 ins_cost(150);
17582 ins_encode %{
17583 __ vmulss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17584 %}
17585 ins_pipe(pipe_slow);
17586 %}
17587
17588 instruct mulD_reg(regD dst, regD src) %{
17589 predicate(UseAVX == 0);
17590 match(Set dst (MulD dst src));
17591
17592 format %{ "mulsd $dst, $src" %}
17593 ins_cost(150);
17594 ins_encode %{
17595 __ mulsd($dst$$XMMRegister, $src$$XMMRegister);
17596 %}
17597 ins_pipe(pipe_slow);
17598 %}
17599
17600 instruct mulD_mem(regD dst, memory src) %{
17601 predicate(UseAVX == 0);
17602 match(Set dst (MulD dst (LoadD src)));
17603
17604 format %{ "mulsd $dst, $src" %}
17605 ins_cost(150);
17606 ins_encode %{
17607 __ mulsd($dst$$XMMRegister, $src$$Address);
17608 %}
17609 ins_pipe(pipe_slow);
17610 %}
17611
17612 instruct mulD_imm(regD dst, immD con) %{
17613 predicate(UseAVX == 0);
17614 match(Set dst (MulD dst con));
17615 format %{ "mulsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17616 ins_cost(150);
17617 ins_encode %{
17618 __ mulsd($dst$$XMMRegister, $constantaddress($con));
17619 %}
17620 ins_pipe(pipe_slow);
17621 %}
17622
17623 instruct mulD_reg_reg(regD dst, regD src1, regD src2) %{
17624 predicate(UseAVX > 0);
17625 match(Set dst (MulD src1 src2));
17626
17627 format %{ "vmulsd $dst, $src1, $src2" %}
17628 ins_cost(150);
17629 ins_encode %{
17630 __ vmulsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17631 %}
17632 ins_pipe(pipe_slow);
17633 %}
17634
17635 instruct mulD_reg_mem(regD dst, regD src1, memory src2) %{
17636 predicate(UseAVX > 0);
17637 match(Set dst (MulD src1 (LoadD src2)));
17638
17639 format %{ "vmulsd $dst, $src1, $src2" %}
17640 ins_cost(150);
17641 ins_encode %{
17642 __ vmulsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17643 %}
17644 ins_pipe(pipe_slow);
17645 %}
17646
17647 instruct mulD_reg_imm(regD dst, regD src, immD con) %{
17648 predicate(UseAVX > 0);
17649 match(Set dst (MulD src con));
17650
17651 format %{ "vmulsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17652 ins_cost(150);
17653 ins_encode %{
17654 __ vmulsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17655 %}
17656 ins_pipe(pipe_slow);
17657 %}
17658
17659 instruct divF_reg(regF dst, regF src) %{
17660 predicate(UseAVX == 0);
17661 match(Set dst (DivF dst src));
17662
17663 format %{ "divss $dst, $src" %}
17664 ins_cost(150);
17665 ins_encode %{
17666 __ divss($dst$$XMMRegister, $src$$XMMRegister);
17667 %}
17668 ins_pipe(pipe_slow);
17669 %}
17670
17671 instruct divF_mem(regF dst, memory src) %{
17672 predicate(UseAVX == 0);
17673 match(Set dst (DivF dst (LoadF src)));
17674
17675 format %{ "divss $dst, $src" %}
17676 ins_cost(150);
17677 ins_encode %{
17678 __ divss($dst$$XMMRegister, $src$$Address);
17679 %}
17680 ins_pipe(pipe_slow);
17681 %}
17682
17683 instruct divF_imm(regF dst, immF con) %{
17684 predicate(UseAVX == 0);
17685 match(Set dst (DivF dst con));
17686 format %{ "divss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
17687 ins_cost(150);
17688 ins_encode %{
17689 __ divss($dst$$XMMRegister, $constantaddress($con));
17690 %}
17691 ins_pipe(pipe_slow);
17692 %}
17693
17694 instruct divF_reg_reg(regF dst, regF src1, regF src2) %{
17695 predicate(UseAVX > 0);
17696 match(Set dst (DivF src1 src2));
17697
17698 format %{ "vdivss $dst, $src1, $src2" %}
17699 ins_cost(150);
17700 ins_encode %{
17701 __ vdivss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17702 %}
17703 ins_pipe(pipe_slow);
17704 %}
17705
17706 instruct divF_reg_mem(regF dst, regF src1, memory src2) %{
17707 predicate(UseAVX > 0);
17708 match(Set dst (DivF src1 (LoadF src2)));
17709
17710 format %{ "vdivss $dst, $src1, $src2" %}
17711 ins_cost(150);
17712 ins_encode %{
17713 __ vdivss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17714 %}
17715 ins_pipe(pipe_slow);
17716 %}
17717
17718 instruct divF_reg_imm(regF dst, regF src, immF con) %{
17719 predicate(UseAVX > 0);
17720 match(Set dst (DivF src con));
17721
17722 format %{ "vdivss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
17723 ins_cost(150);
17724 ins_encode %{
17725 __ vdivss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17726 %}
17727 ins_pipe(pipe_slow);
17728 %}
17729
17730 instruct divD_reg(regD dst, regD src) %{
17731 predicate(UseAVX == 0);
17732 match(Set dst (DivD dst src));
17733
17734 format %{ "divsd $dst, $src" %}
17735 ins_cost(150);
17736 ins_encode %{
17737 __ divsd($dst$$XMMRegister, $src$$XMMRegister);
17738 %}
17739 ins_pipe(pipe_slow);
17740 %}
17741
17742 instruct divD_mem(regD dst, memory src) %{
17743 predicate(UseAVX == 0);
17744 match(Set dst (DivD dst (LoadD src)));
17745
17746 format %{ "divsd $dst, $src" %}
17747 ins_cost(150);
17748 ins_encode %{
17749 __ divsd($dst$$XMMRegister, $src$$Address);
17750 %}
17751 ins_pipe(pipe_slow);
17752 %}
17753
17754 instruct divD_imm(regD dst, immD con) %{
17755 predicate(UseAVX == 0);
17756 match(Set dst (DivD dst con));
17757 format %{ "divsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17758 ins_cost(150);
17759 ins_encode %{
17760 __ divsd($dst$$XMMRegister, $constantaddress($con));
17761 %}
17762 ins_pipe(pipe_slow);
17763 %}
17764
17765 instruct divD_reg_reg(regD dst, regD src1, regD src2) %{
17766 predicate(UseAVX > 0);
17767 match(Set dst (DivD src1 src2));
17768
17769 format %{ "vdivsd $dst, $src1, $src2" %}
17770 ins_cost(150);
17771 ins_encode %{
17772 __ vdivsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17773 %}
17774 ins_pipe(pipe_slow);
17775 %}
17776
17777 instruct divD_reg_mem(regD dst, regD src1, memory src2) %{
17778 predicate(UseAVX > 0);
17779 match(Set dst (DivD src1 (LoadD src2)));
17780
17781 format %{ "vdivsd $dst, $src1, $src2" %}
17782 ins_cost(150);
17783 ins_encode %{
17784 __ vdivsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17785 %}
17786 ins_pipe(pipe_slow);
17787 %}
17788
17789 instruct divD_reg_imm(regD dst, regD src, immD con) %{
17790 predicate(UseAVX > 0);
17791 match(Set dst (DivD src con));
17792
17793 format %{ "vdivsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17794 ins_cost(150);
17795 ins_encode %{
17796 __ vdivsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17797 %}
17798 ins_pipe(pipe_slow);
17799 %}
17800
17801 instruct absF_reg(regF dst) %{
17802 predicate(UseAVX == 0);
17803 match(Set dst (AbsF dst));
17804 ins_cost(150);
17805 format %{ "andps $dst, [0x7fffffff]\t# abs float by sign masking" %}
17806 ins_encode %{
17807 __ andps($dst$$XMMRegister, ExternalAddress(float_signmask()));
17808 %}
17809 ins_pipe(pipe_slow);
17810 %}
17811
17812 instruct absF_reg_reg(vlRegF dst, vlRegF src) %{
17813 predicate(UseAVX > 0);
17814 match(Set dst (AbsF src));
17815 ins_cost(150);
17816 format %{ "vandps $dst, $src, [0x7fffffff]\t# abs float by sign masking" %}
17817 ins_encode %{
17818 int vlen_enc = Assembler::AVX_128bit;
17819 __ vandps($dst$$XMMRegister, $src$$XMMRegister,
17820 ExternalAddress(float_signmask()), vlen_enc);
17821 %}
17822 ins_pipe(pipe_slow);
17823 %}
17824
17825 instruct absD_reg(regD dst) %{
17826 predicate(UseAVX == 0);
17827 match(Set dst (AbsD dst));
17828 ins_cost(150);
17829 format %{ "andpd $dst, [0x7fffffffffffffff]\t"
17830 "# abs double by sign masking" %}
17831 ins_encode %{
17832 __ andpd($dst$$XMMRegister, ExternalAddress(double_signmask()));
17833 %}
17834 ins_pipe(pipe_slow);
17835 %}
17836
17837 instruct absD_reg_reg(vlRegD dst, vlRegD src) %{
17838 predicate(UseAVX > 0);
17839 match(Set dst (AbsD src));
17840 ins_cost(150);
17841 format %{ "vandpd $dst, $src, [0x7fffffffffffffff]\t"
17842 "# abs double by sign masking" %}
17843 ins_encode %{
17844 int vlen_enc = Assembler::AVX_128bit;
17845 __ vandpd($dst$$XMMRegister, $src$$XMMRegister,
17846 ExternalAddress(double_signmask()), vlen_enc);
17847 %}
17848 ins_pipe(pipe_slow);
17849 %}
17850
17851 instruct negF_reg(regF dst) %{
17852 predicate(UseAVX == 0);
17853 match(Set dst (NegF dst));
17854 ins_cost(150);
17855 format %{ "xorps $dst, [0x80000000]\t# neg float by sign flipping" %}
17856 ins_encode %{
17857 __ xorps($dst$$XMMRegister, ExternalAddress(float_signflip()));
17858 %}
17859 ins_pipe(pipe_slow);
17860 %}
17861
17862 instruct negF_reg_reg(vlRegF dst, vlRegF src) %{
17863 predicate(UseAVX > 0);
17864 match(Set dst (NegF src));
17865 ins_cost(150);
17866 format %{ "vnegatess $dst, $src, [0x80000000]\t# neg float by sign flipping" %}
17867 ins_encode %{
17868 __ vnegatess($dst$$XMMRegister, $src$$XMMRegister,
17869 ExternalAddress(float_signflip()));
17870 %}
17871 ins_pipe(pipe_slow);
17872 %}
17873
17874 instruct negD_reg(regD dst) %{
17875 predicate(UseAVX == 0);
17876 match(Set dst (NegD dst));
17877 ins_cost(150);
17878 format %{ "xorpd $dst, [0x8000000000000000]\t"
17879 "# neg double by sign flipping" %}
17880 ins_encode %{
17881 __ xorpd($dst$$XMMRegister, ExternalAddress(double_signflip()));
17882 %}
17883 ins_pipe(pipe_slow);
17884 %}
17885
17886 instruct negD_reg_reg(vlRegD dst, vlRegD src) %{
17887 predicate(UseAVX > 0);
17888 match(Set dst (NegD src));
17889 ins_cost(150);
17890 format %{ "vnegatesd $dst, $src, [0x8000000000000000]\t"
17891 "# neg double by sign flipping" %}
17892 ins_encode %{
17893 __ vnegatesd($dst$$XMMRegister, $src$$XMMRegister,
17894 ExternalAddress(double_signflip()));
17895 %}
17896 ins_pipe(pipe_slow);
17897 %}
17898
17899 // sqrtss instruction needs destination register to be pre initialized for best performance
17900 // Therefore only the instruct rule where the input is pre-loaded into dst register is defined below
17901 instruct sqrtF_reg(regF dst) %{
17902 match(Set dst (SqrtF dst));
17903 format %{ "sqrtss $dst, $dst" %}
17904 ins_encode %{
17905 __ sqrtss($dst$$XMMRegister, $dst$$XMMRegister);
17906 %}
17907 ins_pipe(pipe_slow);
17908 %}
17909
17910 // sqrtsd instruction needs destination register to be pre initialized for best performance
17911 // Therefore only the instruct rule where the input is pre-loaded into dst register is defined below
17912 instruct sqrtD_reg(regD dst) %{
17913 match(Set dst (SqrtD dst));
17914 format %{ "sqrtsd $dst, $dst" %}
17915 ins_encode %{
17916 __ sqrtsd($dst$$XMMRegister, $dst$$XMMRegister);
17917 %}
17918 ins_pipe(pipe_slow);
17919 %}
17920
17921 instruct convF2HF_reg_reg(rRegI dst, vlRegF src, vlRegF tmp) %{
17922 effect(TEMP tmp);
17923 match(Set dst (ConvF2HF src));
17924 ins_cost(125);
17925 format %{ "vcvtps2ph $dst,$src \t using $tmp as TEMP"%}
17926 ins_encode %{
17927 __ flt_to_flt16($dst$$Register, $src$$XMMRegister, $tmp$$XMMRegister);
17928 %}
17929 ins_pipe( pipe_slow );
17930 %}
17931
17932 instruct convF2HF_mem_reg(memory mem, regF src, kReg ktmp, rRegI rtmp) %{
17933 predicate((UseAVX > 2) && VM_Version::supports_avx512vl());
17934 effect(TEMP ktmp, TEMP rtmp);
17935 match(Set mem (StoreC mem (ConvF2HF src)));
17936 format %{ "evcvtps2ph $mem,$src \t using $ktmp and $rtmp as TEMP" %}
17937 ins_encode %{
17938 __ movl($rtmp$$Register, 0x1);
17939 __ kmovwl($ktmp$$KRegister, $rtmp$$Register);
17940 __ evcvtps2ph($mem$$Address, $ktmp$$KRegister, $src$$XMMRegister, 0x04, Assembler::AVX_128bit);
17941 %}
17942 ins_pipe( pipe_slow );
17943 %}
17944
17945 instruct vconvF2HF(vec dst, vec src) %{
17946 match(Set dst (VectorCastF2HF src));
17947 format %{ "vector_conv_F2HF $dst $src" %}
17948 ins_encode %{
17949 int vlen_enc = vector_length_encoding(this, $src);
17950 __ vcvtps2ph($dst$$XMMRegister, $src$$XMMRegister, 0x04, vlen_enc);
17951 %}
17952 ins_pipe( pipe_slow );
17953 %}
17954
17955 instruct vconvF2HF_mem_reg(memory mem, vec src) %{
17956 predicate(n->as_StoreVector()->memory_size() >= 16);
17957 match(Set mem (StoreVector mem (VectorCastF2HF src)));
17958 format %{ "vcvtps2ph $mem,$src" %}
17959 ins_encode %{
17960 int vlen_enc = vector_length_encoding(this, $src);
17961 __ vcvtps2ph($mem$$Address, $src$$XMMRegister, 0x04, vlen_enc);
17962 %}
17963 ins_pipe( pipe_slow );
17964 %}
17965
17966 instruct convHF2F_reg_reg(vlRegF dst, rRegI src) %{
17967 match(Set dst (ConvHF2F src));
17968 format %{ "vcvtph2ps $dst,$src" %}
17969 ins_encode %{
17970 __ flt16_to_flt($dst$$XMMRegister, $src$$Register);
17971 %}
17972 ins_pipe( pipe_slow );
17973 %}
17974
17975 instruct vconvHF2F_reg_mem(vec dst, memory mem) %{
17976 match(Set dst (VectorCastHF2F (LoadVector mem)));
17977 format %{ "vcvtph2ps $dst,$mem" %}
17978 ins_encode %{
17979 int vlen_enc = vector_length_encoding(this);
17980 __ vcvtph2ps($dst$$XMMRegister, $mem$$Address, vlen_enc);
17981 %}
17982 ins_pipe( pipe_slow );
17983 %}
17984
17985 instruct vconvHF2F(vec dst, vec src) %{
17986 match(Set dst (VectorCastHF2F src));
17987 ins_cost(125);
17988 format %{ "vector_conv_HF2F $dst,$src" %}
17989 ins_encode %{
17990 int vlen_enc = vector_length_encoding(this);
17991 __ vcvtph2ps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
17992 %}
17993 ins_pipe( pipe_slow );
17994 %}
17995
17996 // ---------------------------------------- VectorReinterpret ------------------------------------
17997 instruct reinterpret_mask(kReg dst) %{
17998 predicate(n->bottom_type()->isa_pvectmask() &&
17999 Matcher::vector_length(n) == Matcher::vector_length(n->in(1))); // dst == src
18000 match(Set dst (VectorReinterpret dst));
18001 ins_cost(125);
18002 format %{ "vector_reinterpret $dst\t!" %}
18003 ins_encode %{
18004 // empty
18005 %}
18006 ins_pipe( pipe_slow );
18007 %}
18008
18009 instruct reinterpret_mask_W2B(kReg dst, kReg src, vec xtmp) %{
18010 predicate(UseAVX > 2 && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
18011 n->bottom_type()->isa_pvectmask() &&
18012 n->in(1)->bottom_type()->isa_pvectmask() &&
18013 n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_SHORT &&
18014 n->bottom_type()->is_pvectmask()->element_basic_type() == T_BYTE); // dst == src
18015 match(Set dst (VectorReinterpret src));
18016 effect(TEMP xtmp);
18017 format %{ "vector_mask_reinterpret_W2B $dst $src\t!" %}
18018 ins_encode %{
18019 int src_sz = Matcher::vector_length(this, $src)*type2aelembytes(T_SHORT);
18020 int dst_sz = Matcher::vector_length(this)*type2aelembytes(T_BYTE);
18021 assert(src_sz == dst_sz , "src and dst size mismatch");
18022 int vlen_enc = vector_length_encoding(src_sz);
18023 __ evpmovm2w($xtmp$$XMMRegister, $src$$KRegister, vlen_enc);
18024 __ evpmovb2m($dst$$KRegister, $xtmp$$XMMRegister, vlen_enc);
18025 %}
18026 ins_pipe( pipe_slow );
18027 %}
18028
18029 instruct reinterpret_mask_D2B(kReg dst, kReg src, vec xtmp) %{
18030 predicate(UseAVX > 2 && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
18031 n->bottom_type()->isa_pvectmask() &&
18032 n->in(1)->bottom_type()->isa_pvectmask() &&
18033 (n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_INT ||
18034 n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_FLOAT) &&
18035 n->bottom_type()->is_pvectmask()->element_basic_type() == T_BYTE); // dst == src
18036 match(Set dst (VectorReinterpret src));
18037 effect(TEMP xtmp);
18038 format %{ "vector_mask_reinterpret_D2B $dst $src\t!" %}
18039 ins_encode %{
18040 int src_sz = Matcher::vector_length(this, $src)*type2aelembytes(T_INT);
18041 int dst_sz = Matcher::vector_length(this)*type2aelembytes(T_BYTE);
18042 assert(src_sz == dst_sz , "src and dst size mismatch");
18043 int vlen_enc = vector_length_encoding(src_sz);
18044 __ evpmovm2d($xtmp$$XMMRegister, $src$$KRegister, vlen_enc);
18045 __ evpmovb2m($dst$$KRegister, $xtmp$$XMMRegister, vlen_enc);
18046 %}
18047 ins_pipe( pipe_slow );
18048 %}
18049
18050 instruct reinterpret_mask_Q2B(kReg dst, kReg src, vec xtmp) %{
18051 predicate(UseAVX > 2 && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
18052 n->bottom_type()->isa_pvectmask() &&
18053 n->in(1)->bottom_type()->isa_pvectmask() &&
18054 (n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_LONG ||
18055 n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_DOUBLE) &&
18056 n->bottom_type()->is_pvectmask()->element_basic_type() == T_BYTE); // dst == src
18057 match(Set dst (VectorReinterpret src));
18058 effect(TEMP xtmp);
18059 format %{ "vector_mask_reinterpret_Q2B $dst $src\t!" %}
18060 ins_encode %{
18061 int src_sz = Matcher::vector_length(this, $src)*type2aelembytes(T_LONG);
18062 int dst_sz = Matcher::vector_length(this)*type2aelembytes(T_BYTE);
18063 assert(src_sz == dst_sz , "src and dst size mismatch");
18064 int vlen_enc = vector_length_encoding(src_sz);
18065 __ evpmovm2q($xtmp$$XMMRegister, $src$$KRegister, vlen_enc);
18066 __ evpmovb2m($dst$$KRegister, $xtmp$$XMMRegister, vlen_enc);
18067 %}
18068 ins_pipe( pipe_slow );
18069 %}
18070
18071 instruct reinterpret(vec dst) %{
18072 predicate(!n->bottom_type()->isa_pvectmask() &&
18073 Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1))); // dst == src
18074 match(Set dst (VectorReinterpret dst));
18075 ins_cost(125);
18076 format %{ "vector_reinterpret $dst\t!" %}
18077 ins_encode %{
18078 // empty
18079 %}
18080 ins_pipe( pipe_slow );
18081 %}
18082
18083 instruct reinterpret_expand(vec dst, vec src) %{
18084 predicate(UseAVX == 0 &&
18085 (Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
18086 match(Set dst (VectorReinterpret src));
18087 ins_cost(125);
18088 effect(TEMP dst);
18089 format %{ "vector_reinterpret_expand $dst,$src" %}
18090 ins_encode %{
18091 assert(Matcher::vector_length_in_bytes(this) <= 16, "required");
18092 assert(Matcher::vector_length_in_bytes(this, $src) <= 8, "required");
18093
18094 int src_vlen_in_bytes = Matcher::vector_length_in_bytes(this, $src);
18095 if (src_vlen_in_bytes == 4) {
18096 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_32_bit_mask()), noreg);
18097 } else {
18098 assert(src_vlen_in_bytes == 8, "");
18099 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_64_bit_mask()), noreg);
18100 }
18101 __ pand($dst$$XMMRegister, $src$$XMMRegister);
18102 %}
18103 ins_pipe( pipe_slow );
18104 %}
18105
18106 instruct vreinterpret_expand4(legVec dst, vec src) %{
18107 predicate(UseAVX > 0 &&
18108 !n->bottom_type()->isa_pvectmask() &&
18109 (Matcher::vector_length_in_bytes(n->in(1)) == 4) && // src
18110 (Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
18111 match(Set dst (VectorReinterpret src));
18112 ins_cost(125);
18113 format %{ "vector_reinterpret_expand $dst,$src" %}
18114 ins_encode %{
18115 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_32_bit_mask()), 0, noreg);
18116 %}
18117 ins_pipe( pipe_slow );
18118 %}
18119
18120
18121 instruct vreinterpret_expand(legVec dst, vec src) %{
18122 predicate(UseAVX > 0 &&
18123 !n->bottom_type()->isa_pvectmask() &&
18124 (Matcher::vector_length_in_bytes(n->in(1)) > 4) && // src
18125 (Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
18126 match(Set dst (VectorReinterpret src));
18127 ins_cost(125);
18128 format %{ "vector_reinterpret_expand $dst,$src\t!" %}
18129 ins_encode %{
18130 switch (Matcher::vector_length_in_bytes(this, $src)) {
18131 case 8: __ movq ($dst$$XMMRegister, $src$$XMMRegister); break;
18132 case 16: __ movdqu ($dst$$XMMRegister, $src$$XMMRegister); break;
18133 case 32: __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister); break;
18134 default: ShouldNotReachHere();
18135 }
18136 %}
18137 ins_pipe( pipe_slow );
18138 %}
18139
18140 instruct reinterpret_shrink(vec dst, legVec src) %{
18141 predicate(!n->bottom_type()->isa_pvectmask() &&
18142 Matcher::vector_length_in_bytes(n->in(1)) > Matcher::vector_length_in_bytes(n)); // src > dst
18143 match(Set dst (VectorReinterpret src));
18144 ins_cost(125);
18145 format %{ "vector_reinterpret_shrink $dst,$src\t!" %}
18146 ins_encode %{
18147 switch (Matcher::vector_length_in_bytes(this)) {
18148 case 4: __ movfltz($dst$$XMMRegister, $src$$XMMRegister); break;
18149 case 8: __ movq ($dst$$XMMRegister, $src$$XMMRegister); break;
18150 case 16: __ movdqu ($dst$$XMMRegister, $src$$XMMRegister); break;
18151 case 32: __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister); break;
18152 default: ShouldNotReachHere();
18153 }
18154 %}
18155 ins_pipe( pipe_slow );
18156 %}
18157
18158 // ----------------------------------------------------------------------------------------------------
18159
18160 instruct roundD_reg(legRegD dst, legRegD src, immU8 rmode) %{
18161 match(Set dst (RoundDoubleMode src rmode));
18162 format %{ "roundsd $dst,$src" %}
18163 ins_cost(150);
18164 ins_encode %{
18165 assert(UseSSE >= 4, "required");
18166 if ((UseAVX == 0) && ($dst$$XMMRegister != $src$$XMMRegister)) {
18167 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
18168 }
18169 __ roundsd($dst$$XMMRegister, $src$$XMMRegister, $rmode$$constant);
18170 %}
18171 ins_pipe(pipe_slow);
18172 %}
18173
18174 instruct roundD_imm(legRegD dst, immD con, immU8 rmode) %{
18175 match(Set dst (RoundDoubleMode con rmode));
18176 format %{ "roundsd $dst,[$constantaddress]\t# load from constant table: double=$con" %}
18177 ins_cost(150);
18178 ins_encode %{
18179 assert(UseSSE >= 4, "required");
18180 __ roundsd($dst$$XMMRegister, $constantaddress($con), $rmode$$constant, noreg);
18181 %}
18182 ins_pipe(pipe_slow);
18183 %}
18184
18185 instruct vroundD_reg(legVec dst, legVec src, immU8 rmode) %{
18186 predicate(Matcher::vector_length(n) < 8);
18187 match(Set dst (RoundDoubleModeV src rmode));
18188 format %{ "vroundpd $dst,$src,$rmode\t! round packedD" %}
18189 ins_encode %{
18190 assert(UseAVX > 0, "required");
18191 int vlen_enc = vector_length_encoding(this);
18192 __ vroundpd($dst$$XMMRegister, $src$$XMMRegister, $rmode$$constant, vlen_enc);
18193 %}
18194 ins_pipe( pipe_slow );
18195 %}
18196
18197 instruct vround8D_reg(vec dst, vec src, immU8 rmode) %{
18198 predicate(Matcher::vector_length(n) == 8);
18199 match(Set dst (RoundDoubleModeV src rmode));
18200 format %{ "vrndscalepd $dst,$src,$rmode\t! round packed8D" %}
18201 ins_encode %{
18202 assert(UseAVX > 2, "required");
18203 __ vrndscalepd($dst$$XMMRegister, $src$$XMMRegister, $rmode$$constant, Assembler::AVX_512bit);
18204 %}
18205 ins_pipe( pipe_slow );
18206 %}
18207
18208 instruct vroundD_mem(legVec dst, memory mem, immU8 rmode) %{
18209 predicate(Matcher::vector_length(n) < 8);
18210 match(Set dst (RoundDoubleModeV (LoadVector mem) rmode));
18211 format %{ "vroundpd $dst, $mem, $rmode\t! round packedD" %}
18212 ins_encode %{
18213 assert(UseAVX > 0, "required");
18214 int vlen_enc = vector_length_encoding(this);
18215 __ vroundpd($dst$$XMMRegister, $mem$$Address, $rmode$$constant, vlen_enc);
18216 %}
18217 ins_pipe( pipe_slow );
18218 %}
18219
18220 instruct vround8D_mem(vec dst, memory mem, immU8 rmode) %{
18221 predicate(Matcher::vector_length(n) == 8);
18222 match(Set dst (RoundDoubleModeV (LoadVector mem) rmode));
18223 format %{ "vrndscalepd $dst,$mem,$rmode\t! round packed8D" %}
18224 ins_encode %{
18225 assert(UseAVX > 2, "required");
18226 __ vrndscalepd($dst$$XMMRegister, $mem$$Address, $rmode$$constant, Assembler::AVX_512bit);
18227 %}
18228 ins_pipe( pipe_slow );
18229 %}
18230
18231 instruct onspinwait() %{
18232 match(OnSpinWait);
18233 ins_cost(200);
18234
18235 format %{
18236 $$template
18237 $$emit$$"pause\t! membar_onspinwait"
18238 %}
18239 ins_encode %{
18240 __ pause();
18241 %}
18242 ins_pipe(pipe_slow);
18243 %}
18244
18245 // a * b + c
18246 instruct fmaD_reg(regD a, regD b, regD c) %{
18247 match(Set c (FmaD c (Binary a b)));
18248 format %{ "fmasd $a,$b,$c\t# $c = $a * $b + $c" %}
18249 ins_cost(150);
18250 ins_encode %{
18251 assert(UseFMA, "Needs FMA instructions support.");
18252 __ fmad($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister);
18253 %}
18254 ins_pipe( pipe_slow );
18255 %}
18256
18257 // a * b + c
18258 instruct fmaF_reg(regF a, regF b, regF c) %{
18259 match(Set c (FmaF c (Binary a b)));
18260 format %{ "fmass $a,$b,$c\t# $c = $a * $b + $c" %}
18261 ins_cost(150);
18262 ins_encode %{
18263 assert(UseFMA, "Needs FMA instructions support.");
18264 __ fmaf($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister);
18265 %}
18266 ins_pipe( pipe_slow );
18267 %}
18268
18269 // ====================VECTOR INSTRUCTIONS=====================================
18270
18271 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
18272 instruct MoveVec2Leg(legVec dst, vec src) %{
18273 match(Set dst src);
18274 format %{ "" %}
18275 ins_encode %{
18276 ShouldNotReachHere();
18277 %}
18278 ins_pipe( fpu_reg_reg );
18279 %}
18280
18281 instruct MoveLeg2Vec(vec dst, legVec src) %{
18282 match(Set dst src);
18283 format %{ "" %}
18284 ins_encode %{
18285 ShouldNotReachHere();
18286 %}
18287 ins_pipe( fpu_reg_reg );
18288 %}
18289
18290 // ============================================================================
18291
18292 // Load vectors generic operand pattern
18293 instruct loadV(vec dst, memory mem) %{
18294 match(Set dst (LoadVector mem));
18295 ins_cost(125);
18296 format %{ "load_vector $dst,$mem" %}
18297 ins_encode %{
18298 BasicType bt = Matcher::vector_element_basic_type(this);
18299 __ load_vector(bt, $dst$$XMMRegister, $mem$$Address, Matcher::vector_length_in_bytes(this));
18300 %}
18301 ins_pipe( pipe_slow );
18302 %}
18303
18304 // Store vectors generic operand pattern.
18305 instruct storeV(memory mem, vec src) %{
18306 match(Set mem (StoreVector mem src));
18307 ins_cost(145);
18308 format %{ "store_vector $mem,$src\n\t" %}
18309 ins_encode %{
18310 switch (Matcher::vector_length_in_bytes(this, $src)) {
18311 case 4: __ movdl ($mem$$Address, $src$$XMMRegister); break;
18312 case 8: __ movq ($mem$$Address, $src$$XMMRegister); break;
18313 case 16: __ movdqu ($mem$$Address, $src$$XMMRegister); break;
18314 case 32: __ vmovdqu ($mem$$Address, $src$$XMMRegister); break;
18315 case 64: __ evmovdqul($mem$$Address, $src$$XMMRegister, Assembler::AVX_512bit); break;
18316 default: ShouldNotReachHere();
18317 }
18318 %}
18319 ins_pipe( pipe_slow );
18320 %}
18321
18322 // ---------------------------------------- Gather ------------------------------------
18323
18324 // Gather BYTE, SHORT, INT, LONG, FLOAT, DOUBLE
18325
18326 instruct gather(legVec dst, memory mem, legVec idx, rRegP tmp, legVec mask) %{
18327 predicate(!VM_Version::supports_avx512vl() && !is_subword_type(Matcher::vector_element_basic_type(n)) &&
18328 Matcher::vector_length_in_bytes(n) <= 32);
18329 match(Set dst (LoadVectorGather mem idx));
18330 effect(TEMP dst, TEMP tmp, TEMP mask);
18331 format %{ "load_vector_gather $dst, $mem, $idx\t! using $tmp and $mask as TEMP" %}
18332 ins_encode %{
18333 int vlen_enc = vector_length_encoding(this);
18334 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18335 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18336 __ vpcmpeqd($mask$$XMMRegister, $mask$$XMMRegister, $mask$$XMMRegister, vlen_enc);
18337 __ lea($tmp$$Register, $mem$$Address);
18338 __ vgather(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx$$XMMRegister, $mask$$XMMRegister, vlen_enc);
18339 %}
18340 ins_pipe( pipe_slow );
18341 %}
18342
18343
18344 instruct evgather(vec dst, memory mem, vec idx, rRegP tmp, kReg ktmp) %{
18345 predicate((VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64) &&
18346 !is_subword_type(Matcher::vector_element_basic_type(n)));
18347 match(Set dst (LoadVectorGather mem idx));
18348 effect(TEMP dst, TEMP tmp, TEMP ktmp);
18349 format %{ "load_vector_gather $dst, $mem, $idx\t! using $tmp and ktmp as TEMP" %}
18350 ins_encode %{
18351 int vlen_enc = vector_length_encoding(this);
18352 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18353 __ kxnorwl($ktmp$$KRegister, $ktmp$$KRegister, $ktmp$$KRegister);
18354 __ lea($tmp$$Register, $mem$$Address);
18355 __ evgather(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $tmp$$Register, $idx$$XMMRegister, vlen_enc);
18356 %}
18357 ins_pipe( pipe_slow );
18358 %}
18359
18360 instruct evgather_masked(vec dst, memory mem, vec idx, kReg mask, kReg ktmp, rRegP tmp) %{
18361 predicate((VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64) &&
18362 !is_subword_type(Matcher::vector_element_basic_type(n)));
18363 match(Set dst (LoadVectorGatherMasked mem (Binary idx mask)));
18364 effect(TEMP_DEF dst, TEMP tmp, TEMP ktmp);
18365 format %{ "load_vector_gather_masked $dst, $mem, $idx, $mask\t! using $tmp and ktmp as TEMP" %}
18366 ins_encode %{
18367 assert(UseAVX > 2, "sanity");
18368 int vlen_enc = vector_length_encoding(this);
18369 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18370 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18371 // Note: Since gather instruction partially updates the opmask register used
18372 // for predication hense moving mask operand to a temporary.
18373 __ kmovwl($ktmp$$KRegister, $mask$$KRegister);
18374 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18375 __ lea($tmp$$Register, $mem$$Address);
18376 __ evgather(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $tmp$$Register, $idx$$XMMRegister, vlen_enc);
18377 %}
18378 ins_pipe( pipe_slow );
18379 %}
18380
18381 instruct vgather_subwordLE8B(vec dst, memory mem, rRegP idx_base, rRegP tmp, rRegI rtmp) %{
18382 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) <= 8);
18383 match(Set dst (LoadVectorGather mem idx_base));
18384 effect(TEMP tmp, TEMP rtmp);
18385 format %{ "vector_gatherLE8 $dst, $mem, $idx_base\t! using $tmp and $rtmp as TEMP" %}
18386 ins_encode %{
18387 int vlen_enc = vector_length_encoding(this);
18388 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18389 __ lea($tmp$$Register, $mem$$Address);
18390 __ vgather8b(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base$$Register, $rtmp$$Register, vlen_enc);
18391 %}
18392 ins_pipe( pipe_slow );
18393 %}
18394
18395 instruct vgather_subwordGT8B(vec dst, memory mem, rRegP idx_base, rRegP tmp, rRegP idx_base_temp,
18396 vec xtmp1, vec xtmp2, vec xtmp3, rRegI rtmp, rRegI length, rFlagsReg cr) %{
18397 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) > 8);
18398 match(Set dst (LoadVectorGather mem idx_base));
18399 effect(TEMP_DEF dst, TEMP tmp, TEMP idx_base_temp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp, TEMP length, KILL cr);
18400 format %{ "vector_gatherGT8 $dst, $mem, $idx_base\t! using $tmp, $idx_base_temp, $xtmp1, $xtmp2, $xtmp3, $rtmp and $length as TEMP" %}
18401 ins_encode %{
18402 int vlen_enc = vector_length_encoding(this);
18403 int vector_len = Matcher::vector_length(this);
18404 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18405 __ lea($tmp$$Register, $mem$$Address);
18406 __ movptr($idx_base_temp$$Register, $idx_base$$Register);
18407 __ vgather_subword(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base_temp$$Register, noreg, $xtmp1$$XMMRegister,
18408 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, noreg, $length$$Register, vector_len, vlen_enc);
18409 %}
18410 ins_pipe( pipe_slow );
18411 %}
18412
18413 instruct vgather_masked_subwordLE8B_avx3(vec dst, memory mem, rRegP idx_base, kReg mask, rRegL mask_idx, rRegP tmp, rRegI rtmp, rRegL rtmp2, rFlagsReg cr) %{
18414 predicate(VM_Version::supports_avx512bw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) <= 8);
18415 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18416 effect(TEMP mask_idx, TEMP tmp, TEMP rtmp, TEMP rtmp2, KILL cr);
18417 format %{ "vector_masked_gatherLE8 $dst, $mem, $idx_base, $mask\t! using $mask_idx, $tmp, $rtmp and $rtmp2 as TEMP" %}
18418 ins_encode %{
18419 int vlen_enc = vector_length_encoding(this);
18420 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18421 __ xorq($mask_idx$$Register, $mask_idx$$Register);
18422 __ lea($tmp$$Register, $mem$$Address);
18423 __ kmovql($rtmp2$$Register, $mask$$KRegister);
18424 __ vgather8b_masked(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base$$Register, $rtmp2$$Register, $mask_idx$$Register, $rtmp$$Register, vlen_enc);
18425 %}
18426 ins_pipe( pipe_slow );
18427 %}
18428
18429 instruct vgather_masked_subwordGT8B_avx3(vec dst, memory mem, rRegP idx_base, kReg mask, rRegP tmp, rRegP idx_base_temp,
18430 vec xtmp1, vec xtmp2, vec xtmp3, rRegI rtmp, rRegL rtmp2, rRegL mask_idx, rRegI length, rFlagsReg cr) %{
18431 predicate(VM_Version::supports_avx512bw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) > 8);
18432 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18433 effect(TEMP_DEF dst, TEMP tmp, TEMP idx_base_temp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp, TEMP rtmp2, TEMP mask_idx, TEMP length, KILL cr);
18434 format %{ "vector_gatherGT8_masked $dst, $mem, $idx_base, $mask\t! using $tmp, $idx_base_temp, $xtmp1, $xtmp2, $xtmp3, $rtmp, $rtmp2, $mask_idx and $length as TEMP" %}
18435 ins_encode %{
18436 int vlen_enc = vector_length_encoding(this);
18437 int vector_len = Matcher::vector_length(this);
18438 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18439 __ xorq($mask_idx$$Register, $mask_idx$$Register);
18440 __ lea($tmp$$Register, $mem$$Address);
18441 __ movptr($idx_base_temp$$Register, $idx_base$$Register);
18442 __ kmovql($rtmp2$$Register, $mask$$KRegister);
18443 __ vgather_subword(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base_temp$$Register, $rtmp2$$Register, $xtmp1$$XMMRegister,
18444 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, $mask_idx$$Register, $length$$Register, vector_len, vlen_enc);
18445 %}
18446 ins_pipe( pipe_slow );
18447 %}
18448
18449 instruct vgather_masked_subwordLE8B_avx2(vec dst, memory mem, rRegP idx_base, vec mask, rRegI mask_idx, rRegP tmp, rRegI rtmp, rRegI rtmp2, rFlagsReg cr) %{
18450 predicate(!VM_Version::supports_avx512vlbw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) <= 8);
18451 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18452 effect(TEMP mask_idx, TEMP tmp, TEMP rtmp, TEMP rtmp2, KILL cr);
18453 format %{ "vector_masked_gatherLE8 $dst, $mem, $idx_base, $mask\t! using $mask_idx, $tmp, $rtmp and $rtmp2 as TEMP" %}
18454 ins_encode %{
18455 int vlen_enc = vector_length_encoding(this);
18456 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18457 __ lea($tmp$$Register, $mem$$Address);
18458 __ vpmovmskb($rtmp2$$Register, $mask$$XMMRegister, vlen_enc);
18459 if (elem_bt == T_SHORT) {
18460 __ movl($mask_idx$$Register, 0x55555555);
18461 __ pextl($rtmp2$$Register, $rtmp2$$Register, $mask_idx$$Register);
18462 }
18463 __ xorl($mask_idx$$Register, $mask_idx$$Register);
18464 __ vgather8b_masked(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base$$Register, $rtmp2$$Register, $mask_idx$$Register, $rtmp$$Register, vlen_enc);
18465 %}
18466 ins_pipe( pipe_slow );
18467 %}
18468
18469 instruct vgather_masked_subwordGT8B_avx2(vec dst, memory mem, rRegP idx_base, vec mask, rRegP tmp, rRegP idx_base_temp,
18470 vec xtmp1, vec xtmp2, vec xtmp3, rRegI rtmp, rRegI rtmp2, rRegI mask_idx, rRegI length, rFlagsReg cr) %{
18471 predicate(!VM_Version::supports_avx512vlbw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) > 8);
18472 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18473 effect(TEMP_DEF dst, TEMP tmp, TEMP idx_base_temp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp, TEMP rtmp2, TEMP mask_idx, TEMP length, KILL cr);
18474 format %{ "vector_gatherGT8_masked $dst, $mem, $idx_base, $mask\t! using $tmp, $idx_base_temp, $xtmp1, $xtmp2, $xtmp3, $rtmp, $rtmp2, $mask_idx and $length as TEMP" %}
18475 ins_encode %{
18476 int vlen_enc = vector_length_encoding(this);
18477 int vector_len = Matcher::vector_length(this);
18478 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18479 __ lea($tmp$$Register, $mem$$Address);
18480 __ movptr($idx_base_temp$$Register, $idx_base$$Register);
18481 __ vpmovmskb($rtmp2$$Register, $mask$$XMMRegister, vlen_enc);
18482 if (elem_bt == T_SHORT) {
18483 __ movl($mask_idx$$Register, 0x55555555);
18484 __ pextl($rtmp2$$Register, $rtmp2$$Register, $mask_idx$$Register);
18485 }
18486 __ xorl($mask_idx$$Register, $mask_idx$$Register);
18487 __ vgather_subword(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base_temp$$Register, $rtmp2$$Register, $xtmp1$$XMMRegister,
18488 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, $mask_idx$$Register, $length$$Register, vector_len, vlen_enc);
18489 %}
18490 ins_pipe( pipe_slow );
18491 %}
18492
18493 // ====================Scatter=======================================
18494
18495 // Scatter INT, LONG, FLOAT, DOUBLE
18496
18497 instruct scatter(memory mem, vec src, vec idx, rRegP tmp, kReg ktmp) %{
18498 predicate(UseAVX > 2);
18499 match(Set mem (StoreVectorScatter mem (Binary src idx)));
18500 effect(TEMP tmp, TEMP ktmp);
18501 format %{ "store_vector_scatter $mem, $idx, $src\t! using k2 and $tmp as TEMP" %}
18502 ins_encode %{
18503 int vlen_enc = vector_length_encoding(this, $src);
18504 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
18505
18506 assert(Matcher::vector_length_in_bytes(this, $src) >= 16, "sanity");
18507 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18508
18509 __ kmovwl($ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), noreg);
18510 __ lea($tmp$$Register, $mem$$Address);
18511 __ evscatter(elem_bt, $tmp$$Register, $idx$$XMMRegister, $ktmp$$KRegister, $src$$XMMRegister, vlen_enc);
18512 %}
18513 ins_pipe( pipe_slow );
18514 %}
18515
18516 instruct scatter_masked(memory mem, vec src, vec idx, kReg mask, kReg ktmp, rRegP tmp) %{
18517 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx mask))));
18518 effect(TEMP tmp, TEMP ktmp);
18519 format %{ "store_vector_scatter_masked $mem, $idx, $src, $mask\t!" %}
18520 ins_encode %{
18521 int vlen_enc = vector_length_encoding(this, $src);
18522 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
18523 assert(Matcher::vector_length_in_bytes(this, $src) >= 16, "sanity");
18524 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18525 // Note: Since scatter instruction partially updates the opmask register used
18526 // for predication hense moving mask operand to a temporary.
18527 __ kmovwl($ktmp$$KRegister, $mask$$KRegister);
18528 __ lea($tmp$$Register, $mem$$Address);
18529 __ evscatter(elem_bt, $tmp$$Register, $idx$$XMMRegister, $ktmp$$KRegister, $src$$XMMRegister, vlen_enc);
18530 %}
18531 ins_pipe( pipe_slow );
18532 %}
18533
18534 // ====================REPLICATE=======================================
18535
18536 // Replicate byte scalar to be vector
18537 instruct vReplB_reg(vec dst, rRegI src) %{
18538 predicate(Matcher::vector_element_basic_type(n) == T_BYTE);
18539 match(Set dst (Replicate src));
18540 format %{ "replicateB $dst,$src" %}
18541 ins_encode %{
18542 uint vlen = Matcher::vector_length(this);
18543 if (UseAVX >= 2) {
18544 int vlen_enc = vector_length_encoding(this);
18545 if (vlen == 64 || VM_Version::supports_avx512vlbw()) { // AVX512VL for <512bit operands
18546 assert(VM_Version::supports_avx512bw(), "required"); // 512-bit byte vectors assume AVX512BW
18547 __ evpbroadcastb($dst$$XMMRegister, $src$$Register, vlen_enc);
18548 } else {
18549 __ movdl($dst$$XMMRegister, $src$$Register);
18550 __ vpbroadcastb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18551 }
18552 } else {
18553 assert(UseAVX < 2, "");
18554 __ movdl($dst$$XMMRegister, $src$$Register);
18555 __ punpcklbw($dst$$XMMRegister, $dst$$XMMRegister);
18556 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18557 if (vlen >= 16) {
18558 assert(vlen == 16, "");
18559 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18560 }
18561 }
18562 %}
18563 ins_pipe( pipe_slow );
18564 %}
18565
18566 instruct ReplB_mem(vec dst, memory mem) %{
18567 predicate(UseAVX >= 2 && Matcher::vector_element_basic_type(n) == T_BYTE);
18568 match(Set dst (Replicate (LoadB mem)));
18569 format %{ "replicateB $dst,$mem" %}
18570 ins_encode %{
18571 int vlen_enc = vector_length_encoding(this);
18572 __ vpbroadcastb($dst$$XMMRegister, $mem$$Address, vlen_enc);
18573 %}
18574 ins_pipe( pipe_slow );
18575 %}
18576
18577 // ====================ReplicateS=======================================
18578
18579 instruct vReplS_reg(vec dst, rRegI src) %{
18580 predicate(Matcher::vector_element_basic_type(n) == T_SHORT);
18581 match(Set dst (Replicate src));
18582 format %{ "replicateS $dst,$src" %}
18583 ins_encode %{
18584 uint vlen = Matcher::vector_length(this);
18585 int vlen_enc = vector_length_encoding(this);
18586 if (UseAVX >= 2) {
18587 if (vlen == 32 || VM_Version::supports_avx512vlbw()) { // AVX512VL for <512bit operands
18588 assert(VM_Version::supports_avx512bw(), "required"); // 512-bit short vectors assume AVX512BW
18589 __ evpbroadcastw($dst$$XMMRegister, $src$$Register, vlen_enc);
18590 } else {
18591 __ movdl($dst$$XMMRegister, $src$$Register);
18592 __ vpbroadcastw($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18593 }
18594 } else {
18595 assert(UseAVX < 2, "");
18596 __ movdl($dst$$XMMRegister, $src$$Register);
18597 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18598 if (vlen >= 8) {
18599 assert(vlen == 8, "");
18600 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18601 }
18602 }
18603 %}
18604 ins_pipe( pipe_slow );
18605 %}
18606
18607 instruct ReplHF_imm(vec dst, immH con, rRegI rtmp) %{
18608 match(Set dst (Replicate con));
18609 effect(TEMP rtmp);
18610 format %{ "replicateHF $dst, $con \t! using $rtmp as TEMP" %}
18611 ins_encode %{
18612 int vlen_enc = vector_length_encoding(this);
18613 BasicType bt = Matcher::vector_element_basic_type(this);
18614 assert(VM_Version::supports_avx512_fp16() && bt == T_SHORT, "");
18615 __ movl($rtmp$$Register, $con$$constant);
18616 __ evpbroadcastw($dst$$XMMRegister, $rtmp$$Register, vlen_enc);
18617 %}
18618 ins_pipe( pipe_slow );
18619 %}
18620
18621 instruct ReplHF_reg(vec dst, regF src, rRegI rtmp) %{
18622 predicate(VM_Version::supports_avx512_fp16() && Matcher::vector_element_basic_type(n) == T_SHORT);
18623 match(Set dst (Replicate src));
18624 effect(TEMP rtmp);
18625 format %{ "replicateHF $dst, $src \t! using $rtmp as TEMP" %}
18626 ins_encode %{
18627 int vlen_enc = vector_length_encoding(this);
18628 __ evmovw($rtmp$$Register, $src$$XMMRegister);
18629 __ evpbroadcastw($dst$$XMMRegister, $rtmp$$Register, vlen_enc);
18630 %}
18631 ins_pipe( pipe_slow );
18632 %}
18633
18634 instruct ReplS_mem(vec dst, memory mem) %{
18635 predicate(UseAVX >= 2 && Matcher::vector_element_basic_type(n) == T_SHORT);
18636 match(Set dst (Replicate (LoadS mem)));
18637 format %{ "replicateS $dst,$mem" %}
18638 ins_encode %{
18639 int vlen_enc = vector_length_encoding(this);
18640 __ vpbroadcastw($dst$$XMMRegister, $mem$$Address, vlen_enc);
18641 %}
18642 ins_pipe( pipe_slow );
18643 %}
18644
18645 // ====================ReplicateI=======================================
18646
18647 instruct ReplI_reg(vec dst, rRegI src) %{
18648 predicate(Matcher::vector_element_basic_type(n) == T_INT);
18649 match(Set dst (Replicate src));
18650 format %{ "replicateI $dst,$src" %}
18651 ins_encode %{
18652 uint vlen = Matcher::vector_length(this);
18653 int vlen_enc = vector_length_encoding(this);
18654 if (vlen == 16 || VM_Version::supports_avx512vl()) { // AVX512VL for <512bit operands
18655 __ evpbroadcastd($dst$$XMMRegister, $src$$Register, vlen_enc);
18656 } else if (VM_Version::supports_avx2()) {
18657 __ movdl($dst$$XMMRegister, $src$$Register);
18658 __ vpbroadcastd($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18659 } else {
18660 __ movdl($dst$$XMMRegister, $src$$Register);
18661 __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18662 }
18663 %}
18664 ins_pipe( pipe_slow );
18665 %}
18666
18667 instruct ReplI_mem(vec dst, memory mem) %{
18668 predicate(Matcher::vector_element_basic_type(n) == T_INT);
18669 match(Set dst (Replicate (LoadI mem)));
18670 format %{ "replicateI $dst,$mem" %}
18671 ins_encode %{
18672 int vlen_enc = vector_length_encoding(this);
18673 if (VM_Version::supports_avx2()) {
18674 __ vpbroadcastd($dst$$XMMRegister, $mem$$Address, vlen_enc);
18675 } else if (VM_Version::supports_avx()) {
18676 __ vbroadcastss($dst$$XMMRegister, $mem$$Address, vlen_enc);
18677 } else {
18678 __ movdl($dst$$XMMRegister, $mem$$Address);
18679 __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18680 }
18681 %}
18682 ins_pipe( pipe_slow );
18683 %}
18684
18685 instruct ReplI_imm(vec dst, immI con) %{
18686 predicate(Matcher::is_non_long_integral_vector(n));
18687 match(Set dst (Replicate con));
18688 format %{ "replicateI $dst,$con" %}
18689 ins_encode %{
18690 InternalAddress addr = $constantaddress(vreplicate_imm(Matcher::vector_element_basic_type(this), $con$$constant,
18691 (VM_Version::supports_sse3() ? (VM_Version::supports_avx() ? 4 : 8) : 16) /
18692 type2aelembytes(Matcher::vector_element_basic_type(this))));
18693 BasicType bt = Matcher::vector_element_basic_type(this);
18694 int vlen = Matcher::vector_length_in_bytes(this);
18695 __ load_constant_vector(bt, $dst$$XMMRegister, addr, vlen);
18696 %}
18697 ins_pipe( pipe_slow );
18698 %}
18699
18700 // Replicate scalar zero to be vector
18701 instruct ReplI_zero(vec dst, immI_0 zero) %{
18702 predicate(Matcher::is_non_long_integral_vector(n));
18703 match(Set dst (Replicate zero));
18704 format %{ "replicateI $dst,$zero" %}
18705 ins_encode %{
18706 int vlen_enc = vector_length_encoding(this);
18707 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vl()) {
18708 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18709 } else {
18710 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
18711 }
18712 %}
18713 ins_pipe( fpu_reg_reg );
18714 %}
18715
18716 instruct ReplI_M1(vec dst, immI_M1 con) %{
18717 predicate(Matcher::is_non_long_integral_vector(n));
18718 match(Set dst (Replicate con));
18719 format %{ "vallones $dst" %}
18720 ins_encode %{
18721 int vector_len = vector_length_encoding(this);
18722 __ vallones($dst$$XMMRegister, vector_len);
18723 %}
18724 ins_pipe( pipe_slow );
18725 %}
18726
18727 // ====================ReplicateL=======================================
18728
18729 // Replicate long (8 byte) scalar to be vector
18730 instruct ReplL_reg(vec dst, rRegL src) %{
18731 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18732 match(Set dst (Replicate src));
18733 format %{ "replicateL $dst,$src" %}
18734 ins_encode %{
18735 int vlen = Matcher::vector_length(this);
18736 int vlen_enc = vector_length_encoding(this);
18737 if (vlen == 8 || VM_Version::supports_avx512vl()) { // AVX512VL for <512bit operands
18738 __ evpbroadcastq($dst$$XMMRegister, $src$$Register, vlen_enc);
18739 } else if (VM_Version::supports_avx2()) {
18740 __ movdq($dst$$XMMRegister, $src$$Register);
18741 __ vpbroadcastq($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18742 } else {
18743 __ movdq($dst$$XMMRegister, $src$$Register);
18744 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18745 }
18746 %}
18747 ins_pipe( pipe_slow );
18748 %}
18749
18750 instruct ReplL_mem(vec dst, memory mem) %{
18751 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18752 match(Set dst (Replicate (LoadL mem)));
18753 format %{ "replicateL $dst,$mem" %}
18754 ins_encode %{
18755 int vlen_enc = vector_length_encoding(this);
18756 if (VM_Version::supports_avx2()) {
18757 __ vpbroadcastq($dst$$XMMRegister, $mem$$Address, vlen_enc);
18758 } else if (VM_Version::supports_sse3()) {
18759 __ movddup($dst$$XMMRegister, $mem$$Address);
18760 } else {
18761 __ movq($dst$$XMMRegister, $mem$$Address);
18762 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18763 }
18764 %}
18765 ins_pipe( pipe_slow );
18766 %}
18767
18768 // Replicate long (8 byte) scalar immediate to be vector by loading from const table.
18769 instruct ReplL_imm(vec dst, immL con) %{
18770 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18771 match(Set dst (Replicate con));
18772 format %{ "replicateL $dst,$con" %}
18773 ins_encode %{
18774 InternalAddress addr = $constantaddress(vreplicate_imm(T_LONG, $con$$constant, VM_Version::supports_sse3() ? 1 : 2));
18775 int vlen = Matcher::vector_length_in_bytes(this);
18776 __ load_constant_vector(T_LONG, $dst$$XMMRegister, addr, vlen);
18777 %}
18778 ins_pipe( pipe_slow );
18779 %}
18780
18781 instruct ReplL_zero(vec dst, immL0 zero) %{
18782 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18783 match(Set dst (Replicate zero));
18784 format %{ "replicateL $dst,$zero" %}
18785 ins_encode %{
18786 int vlen_enc = vector_length_encoding(this);
18787 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vl()) {
18788 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18789 } else {
18790 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
18791 }
18792 %}
18793 ins_pipe( fpu_reg_reg );
18794 %}
18795
18796 instruct ReplL_M1(vec dst, immL_M1 con) %{
18797 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18798 match(Set dst (Replicate con));
18799 format %{ "vallones $dst" %}
18800 ins_encode %{
18801 int vector_len = vector_length_encoding(this);
18802 __ vallones($dst$$XMMRegister, vector_len);
18803 %}
18804 ins_pipe( pipe_slow );
18805 %}
18806
18807 // ====================ReplicateF=======================================
18808
18809 instruct vReplF_reg(vec dst, vlRegF src) %{
18810 predicate(UseAVX > 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
18811 match(Set dst (Replicate src));
18812 format %{ "replicateF $dst,$src" %}
18813 ins_encode %{
18814 uint vlen = Matcher::vector_length(this);
18815 int vlen_enc = vector_length_encoding(this);
18816 if (vlen <= 4) {
18817 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 0x00, Assembler::AVX_128bit);
18818 } else if (VM_Version::supports_avx2()) {
18819 __ vbroadcastss($dst$$XMMRegister, $src$$XMMRegister, vlen_enc); // reg-to-reg variant requires AVX2
18820 } else {
18821 assert(vlen == 8, "sanity");
18822 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 0x00, Assembler::AVX_128bit);
18823 __ vinsertf128_high($dst$$XMMRegister, $dst$$XMMRegister);
18824 }
18825 %}
18826 ins_pipe( pipe_slow );
18827 %}
18828
18829 instruct ReplF_reg(vec dst, vlRegF src) %{
18830 predicate(UseAVX == 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
18831 match(Set dst (Replicate src));
18832 format %{ "replicateF $dst,$src" %}
18833 ins_encode %{
18834 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x00);
18835 %}
18836 ins_pipe( pipe_slow );
18837 %}
18838
18839 instruct ReplF_mem(vec dst, memory mem) %{
18840 predicate(UseAVX > 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
18841 match(Set dst (Replicate (LoadF mem)));
18842 format %{ "replicateF $dst,$mem" %}
18843 ins_encode %{
18844 int vlen_enc = vector_length_encoding(this);
18845 __ vbroadcastss($dst$$XMMRegister, $mem$$Address, vlen_enc);
18846 %}
18847 ins_pipe( pipe_slow );
18848 %}
18849
18850 // Replicate float scalar immediate to be vector by loading from const table.
18851 instruct ReplF_imm(vec dst, immF con) %{
18852 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
18853 match(Set dst (Replicate con));
18854 format %{ "replicateF $dst,$con" %}
18855 ins_encode %{
18856 InternalAddress addr = $constantaddress(vreplicate_imm(T_FLOAT, $con$$constant,
18857 VM_Version::supports_sse3() ? (VM_Version::supports_avx() ? 1 : 2) : 4));
18858 int vlen = Matcher::vector_length_in_bytes(this);
18859 __ load_constant_vector(T_FLOAT, $dst$$XMMRegister, addr, vlen);
18860 %}
18861 ins_pipe( pipe_slow );
18862 %}
18863
18864 instruct ReplF_zero(vec dst, immF0 zero) %{
18865 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
18866 match(Set dst (Replicate zero));
18867 format %{ "replicateF $dst,$zero" %}
18868 ins_encode %{
18869 int vlen_enc = vector_length_encoding(this);
18870 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vldq()) {
18871 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18872 } else {
18873 __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
18874 }
18875 %}
18876 ins_pipe( fpu_reg_reg );
18877 %}
18878
18879 // ====================ReplicateD=======================================
18880
18881 // Replicate double (8 bytes) scalar to be vector
18882 instruct vReplD_reg(vec dst, vlRegD src) %{
18883 predicate(UseSSE >= 3 && Matcher::vector_element_basic_type(n) == T_DOUBLE);
18884 match(Set dst (Replicate src));
18885 format %{ "replicateD $dst,$src" %}
18886 ins_encode %{
18887 uint vlen = Matcher::vector_length(this);
18888 int vlen_enc = vector_length_encoding(this);
18889 if (vlen <= 2) {
18890 __ movddup($dst$$XMMRegister, $src$$XMMRegister);
18891 } else if (VM_Version::supports_avx2()) {
18892 __ vbroadcastsd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc); // reg-to-reg variant requires AVX2
18893 } else {
18894 assert(vlen == 4, "sanity");
18895 __ movddup($dst$$XMMRegister, $src$$XMMRegister);
18896 __ vinsertf128_high($dst$$XMMRegister, $dst$$XMMRegister);
18897 }
18898 %}
18899 ins_pipe( pipe_slow );
18900 %}
18901
18902 instruct ReplD_reg(vec dst, vlRegD src) %{
18903 predicate(UseSSE < 3 && Matcher::vector_element_basic_type(n) == T_DOUBLE);
18904 match(Set dst (Replicate src));
18905 format %{ "replicateD $dst,$src" %}
18906 ins_encode %{
18907 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x44);
18908 %}
18909 ins_pipe( pipe_slow );
18910 %}
18911
18912 instruct ReplD_mem(vec dst, memory mem) %{
18913 predicate(UseSSE >= 3 && Matcher::vector_element_basic_type(n) == T_DOUBLE);
18914 match(Set dst (Replicate (LoadD mem)));
18915 format %{ "replicateD $dst,$mem" %}
18916 ins_encode %{
18917 if (Matcher::vector_length(this) >= 4) {
18918 int vlen_enc = vector_length_encoding(this);
18919 __ vbroadcastsd($dst$$XMMRegister, $mem$$Address, vlen_enc);
18920 } else {
18921 __ movddup($dst$$XMMRegister, $mem$$Address);
18922 }
18923 %}
18924 ins_pipe( pipe_slow );
18925 %}
18926
18927 // Replicate double (8 byte) scalar immediate to be vector by loading from const table.
18928 instruct ReplD_imm(vec dst, immD con) %{
18929 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
18930 match(Set dst (Replicate con));
18931 format %{ "replicateD $dst,$con" %}
18932 ins_encode %{
18933 InternalAddress addr = $constantaddress(vreplicate_imm(T_DOUBLE, $con$$constant, VM_Version::supports_sse3() ? 1 : 2));
18934 int vlen = Matcher::vector_length_in_bytes(this);
18935 __ load_constant_vector(T_DOUBLE, $dst$$XMMRegister, addr, vlen);
18936 %}
18937 ins_pipe( pipe_slow );
18938 %}
18939
18940 instruct ReplD_zero(vec dst, immD0 zero) %{
18941 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
18942 match(Set dst (Replicate zero));
18943 format %{ "replicateD $dst,$zero" %}
18944 ins_encode %{
18945 int vlen_enc = vector_length_encoding(this);
18946 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vldq()) {
18947 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18948 } else {
18949 __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
18950 }
18951 %}
18952 ins_pipe( fpu_reg_reg );
18953 %}
18954
18955 // ====================VECTOR INSERT=======================================
18956
18957 instruct insert(vec dst, rRegI val, immU8 idx) %{
18958 predicate(Matcher::vector_length_in_bytes(n) < 32);
18959 match(Set dst (VectorInsert (Binary dst val) idx));
18960 format %{ "vector_insert $dst,$val,$idx" %}
18961 ins_encode %{
18962 assert(UseSSE >= 4, "required");
18963 assert(Matcher::vector_length_in_bytes(this) >= 8, "required");
18964
18965 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18966
18967 assert(is_integral_type(elem_bt), "");
18968 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18969
18970 __ insert(elem_bt, $dst$$XMMRegister, $val$$Register, $idx$$constant);
18971 %}
18972 ins_pipe( pipe_slow );
18973 %}
18974
18975 instruct insert32(vec dst, vec src, rRegI val, immU8 idx, vec vtmp) %{
18976 predicate(Matcher::vector_length_in_bytes(n) == 32);
18977 match(Set dst (VectorInsert (Binary src val) idx));
18978 effect(TEMP vtmp);
18979 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18980 ins_encode %{
18981 int vlen_enc = Assembler::AVX_256bit;
18982 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18983 int elem_per_lane = 16/type2aelembytes(elem_bt);
18984 int log2epr = log2(elem_per_lane);
18985
18986 assert(is_integral_type(elem_bt), "sanity");
18987 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18988
18989 uint x_idx = $idx$$constant & right_n_bits(log2epr);
18990 uint y_idx = ($idx$$constant >> log2epr) & 1;
18991 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18992 __ vinsert(elem_bt, $vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
18993 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18994 %}
18995 ins_pipe( pipe_slow );
18996 %}
18997
18998 instruct insert64(vec dst, vec src, rRegI val, immU8 idx, legVec vtmp) %{
18999 predicate(Matcher::vector_length_in_bytes(n) == 64);
19000 match(Set dst (VectorInsert (Binary src val) idx));
19001 effect(TEMP vtmp);
19002 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
19003 ins_encode %{
19004 assert(UseAVX > 2, "sanity");
19005
19006 BasicType elem_bt = Matcher::vector_element_basic_type(this);
19007 int elem_per_lane = 16/type2aelembytes(elem_bt);
19008 int log2epr = log2(elem_per_lane);
19009
19010 assert(is_integral_type(elem_bt), "");
19011 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
19012
19013 uint x_idx = $idx$$constant & right_n_bits(log2epr);
19014 uint y_idx = ($idx$$constant >> log2epr) & 3;
19015 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
19016 __ vinsert(elem_bt, $vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
19017 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
19018 %}
19019 ins_pipe( pipe_slow );
19020 %}
19021
19022 instruct insert2L(vec dst, rRegL val, immU8 idx) %{
19023 predicate(Matcher::vector_length(n) == 2);
19024 match(Set dst (VectorInsert (Binary dst val) idx));
19025 format %{ "vector_insert $dst,$val,$idx" %}
19026 ins_encode %{
19027 assert(UseSSE >= 4, "required");
19028 assert(Matcher::vector_element_basic_type(this) == T_LONG, "");
19029 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
19030
19031 __ pinsrq($dst$$XMMRegister, $val$$Register, $idx$$constant);
19032 %}
19033 ins_pipe( pipe_slow );
19034 %}
19035
19036 instruct insert4L(vec dst, vec src, rRegL val, immU8 idx, vec vtmp) %{
19037 predicate(Matcher::vector_length(n) == 4);
19038 match(Set dst (VectorInsert (Binary src val) idx));
19039 effect(TEMP vtmp);
19040 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
19041 ins_encode %{
19042 assert(Matcher::vector_element_basic_type(this) == T_LONG, "");
19043 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
19044
19045 uint x_idx = $idx$$constant & right_n_bits(1);
19046 uint y_idx = ($idx$$constant >> 1) & 1;
19047 int vlen_enc = Assembler::AVX_256bit;
19048 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
19049 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
19050 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
19051 %}
19052 ins_pipe( pipe_slow );
19053 %}
19054
19055 instruct insert8L(vec dst, vec src, rRegL val, immU8 idx, legVec vtmp) %{
19056 predicate(Matcher::vector_length(n) == 8);
19057 match(Set dst (VectorInsert (Binary src val) idx));
19058 effect(TEMP vtmp);
19059 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
19060 ins_encode %{
19061 assert(Matcher::vector_element_basic_type(this) == T_LONG, "sanity");
19062 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
19063
19064 uint x_idx = $idx$$constant & right_n_bits(1);
19065 uint y_idx = ($idx$$constant >> 1) & 3;
19066 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
19067 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
19068 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
19069 %}
19070 ins_pipe( pipe_slow );
19071 %}
19072
19073 instruct insertF(vec dst, regF val, immU8 idx) %{
19074 predicate(Matcher::vector_length(n) < 8);
19075 match(Set dst (VectorInsert (Binary dst val) idx));
19076 format %{ "vector_insert $dst,$val,$idx" %}
19077 ins_encode %{
19078 assert(UseSSE >= 4, "sanity");
19079
19080 assert(Matcher::vector_element_basic_type(this) == T_FLOAT, "sanity");
19081 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
19082
19083 uint x_idx = $idx$$constant & right_n_bits(2);
19084 __ insertps($dst$$XMMRegister, $val$$XMMRegister, x_idx << 4);
19085 %}
19086 ins_pipe( pipe_slow );
19087 %}
19088
19089 instruct vinsertF(vec dst, vec src, regF val, immU8 idx, vec vtmp) %{
19090 predicate(Matcher::vector_length(n) >= 8);
19091 match(Set dst (VectorInsert (Binary src val) idx));
19092 effect(TEMP vtmp);
19093 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
19094 ins_encode %{
19095 assert(Matcher::vector_element_basic_type(this) == T_FLOAT, "sanity");
19096 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
19097
19098 int vlen = Matcher::vector_length(this);
19099 uint x_idx = $idx$$constant & right_n_bits(2);
19100 if (vlen == 8) {
19101 uint y_idx = ($idx$$constant >> 2) & 1;
19102 int vlen_enc = Assembler::AVX_256bit;
19103 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
19104 __ vinsertps($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$XMMRegister, x_idx << 4);
19105 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
19106 } else {
19107 assert(vlen == 16, "sanity");
19108 uint y_idx = ($idx$$constant >> 2) & 3;
19109 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
19110 __ vinsertps($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$XMMRegister, x_idx << 4);
19111 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
19112 }
19113 %}
19114 ins_pipe( pipe_slow );
19115 %}
19116
19117 instruct insert2D(vec dst, regD val, immU8 idx, rRegL tmp) %{
19118 predicate(Matcher::vector_length(n) == 2);
19119 match(Set dst (VectorInsert (Binary dst val) idx));
19120 effect(TEMP tmp);
19121 format %{ "vector_insert $dst,$val,$idx\t!using $tmp as TEMP" %}
19122 ins_encode %{
19123 assert(UseSSE >= 4, "sanity");
19124 assert(Matcher::vector_element_basic_type(this) == T_DOUBLE, "sanity");
19125 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
19126
19127 __ movq($tmp$$Register, $val$$XMMRegister);
19128 __ pinsrq($dst$$XMMRegister, $tmp$$Register, $idx$$constant);
19129 %}
19130 ins_pipe( pipe_slow );
19131 %}
19132
19133 instruct insert4D(vec dst, vec src, regD val, immU8 idx, rRegL tmp, vec vtmp) %{
19134 predicate(Matcher::vector_length(n) == 4);
19135 match(Set dst (VectorInsert (Binary src val) idx));
19136 effect(TEMP vtmp, TEMP tmp);
19137 format %{ "vector_insert $dst,$src,$val,$idx\t!using $tmp, $vtmp as TEMP" %}
19138 ins_encode %{
19139 assert(Matcher::vector_element_basic_type(this) == T_DOUBLE, "sanity");
19140 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
19141
19142 uint x_idx = $idx$$constant & right_n_bits(1);
19143 uint y_idx = ($idx$$constant >> 1) & 1;
19144 int vlen_enc = Assembler::AVX_256bit;
19145 __ movq($tmp$$Register, $val$$XMMRegister);
19146 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
19147 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $tmp$$Register, x_idx);
19148 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
19149 %}
19150 ins_pipe( pipe_slow );
19151 %}
19152
19153 instruct insert8D(vec dst, vec src, regD val, immI idx, rRegL tmp, legVec vtmp) %{
19154 predicate(Matcher::vector_length(n) == 8);
19155 match(Set dst (VectorInsert (Binary src val) idx));
19156 effect(TEMP tmp, TEMP vtmp);
19157 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
19158 ins_encode %{
19159 assert(Matcher::vector_element_basic_type(this) == T_DOUBLE, "sanity");
19160 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
19161
19162 uint x_idx = $idx$$constant & right_n_bits(1);
19163 uint y_idx = ($idx$$constant >> 1) & 3;
19164 __ movq($tmp$$Register, $val$$XMMRegister);
19165 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
19166 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $tmp$$Register, x_idx);
19167 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
19168 %}
19169 ins_pipe( pipe_slow );
19170 %}
19171
19172 // ====================REDUCTION ARITHMETIC=======================================
19173
19174 // =======================Int Reduction==========================================
19175
19176 instruct reductionI(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19177 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_INT); // src2
19178 match(Set dst (AddReductionVI src1 src2));
19179 match(Set dst (MulReductionVI src1 src2));
19180 match(Set dst (AndReductionV src1 src2));
19181 match(Set dst ( OrReductionV src1 src2));
19182 match(Set dst (XorReductionV src1 src2));
19183 match(Set dst (MinReductionV src1 src2));
19184 match(Set dst (MaxReductionV src1 src2));
19185 match(Set dst (UMinReductionV src1 src2));
19186 match(Set dst (UMaxReductionV src1 src2));
19187 effect(TEMP vtmp1, TEMP vtmp2);
19188 format %{ "vector_reduction_int $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19189 ins_encode %{
19190 int opcode = this->ideal_Opcode();
19191 int vlen = Matcher::vector_length(this, $src2);
19192 __ reduceI(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19193 %}
19194 ins_pipe( pipe_slow );
19195 %}
19196
19197 // =======================Long Reduction==========================================
19198
19199 instruct reductionL(rRegL dst, rRegL src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19200 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG && !VM_Version::supports_avx512dq());
19201 match(Set dst (AddReductionVL src1 src2));
19202 match(Set dst (MulReductionVL src1 src2));
19203 match(Set dst (AndReductionV src1 src2));
19204 match(Set dst ( OrReductionV src1 src2));
19205 match(Set dst (XorReductionV src1 src2));
19206 match(Set dst (MinReductionV src1 src2));
19207 match(Set dst (MaxReductionV src1 src2));
19208 match(Set dst (UMinReductionV src1 src2));
19209 match(Set dst (UMaxReductionV src1 src2));
19210 effect(TEMP vtmp1, TEMP vtmp2);
19211 format %{ "vector_reduction_long $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19212 ins_encode %{
19213 int opcode = this->ideal_Opcode();
19214 int vlen = Matcher::vector_length(this, $src2);
19215 __ reduceL(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19216 %}
19217 ins_pipe( pipe_slow );
19218 %}
19219
19220 instruct reductionL_avx512dq(rRegL dst, rRegL src1, vec src2, vec vtmp1, vec vtmp2) %{
19221 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG && VM_Version::supports_avx512dq());
19222 match(Set dst (AddReductionVL src1 src2));
19223 match(Set dst (MulReductionVL src1 src2));
19224 match(Set dst (AndReductionV src1 src2));
19225 match(Set dst ( OrReductionV src1 src2));
19226 match(Set dst (XorReductionV src1 src2));
19227 match(Set dst (MinReductionV src1 src2));
19228 match(Set dst (MaxReductionV src1 src2));
19229 match(Set dst (UMinReductionV src1 src2));
19230 match(Set dst (UMaxReductionV src1 src2));
19231 effect(TEMP vtmp1, TEMP vtmp2);
19232 format %{ "vector_reduction_long $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19233 ins_encode %{
19234 int opcode = this->ideal_Opcode();
19235 int vlen = Matcher::vector_length(this, $src2);
19236 __ reduceL(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19237 %}
19238 ins_pipe( pipe_slow );
19239 %}
19240
19241 // =======================Float Reduction==========================================
19242
19243 instruct reductionF128(regF dst, vec src, vec vtmp) %{
19244 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) <= 4); // src
19245 match(Set dst (AddReductionVF dst src));
19246 match(Set dst (MulReductionVF dst src));
19247 effect(TEMP dst, TEMP vtmp);
19248 format %{ "vector_reduction_float $dst,$src ; using $vtmp as TEMP" %}
19249 ins_encode %{
19250 int opcode = this->ideal_Opcode();
19251 int vlen = Matcher::vector_length(this, $src);
19252 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister);
19253 %}
19254 ins_pipe( pipe_slow );
19255 %}
19256
19257 instruct reduction8F(regF dst, vec src, vec vtmp1, vec vtmp2) %{
19258 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src
19259 match(Set dst (AddReductionVF dst src));
19260 match(Set dst (MulReductionVF dst src));
19261 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19262 format %{ "vector_reduction_float $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
19263 ins_encode %{
19264 int opcode = this->ideal_Opcode();
19265 int vlen = Matcher::vector_length(this, $src);
19266 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19267 %}
19268 ins_pipe( pipe_slow );
19269 %}
19270
19271 instruct reduction16F(regF dst, legVec src, legVec vtmp1, legVec vtmp2) %{
19272 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 16); // src
19273 match(Set dst (AddReductionVF dst src));
19274 match(Set dst (MulReductionVF dst src));
19275 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19276 format %{ "vector_reduction_float $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
19277 ins_encode %{
19278 int opcode = this->ideal_Opcode();
19279 int vlen = Matcher::vector_length(this, $src);
19280 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19281 %}
19282 ins_pipe( pipe_slow );
19283 %}
19284
19285
19286 instruct unordered_reduction2F(regF dst, regF src1, vec src2) %{
19287 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
19288 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19289 // src1 contains reduction identity
19290 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 2); // src2
19291 match(Set dst (AddReductionVF src1 src2));
19292 match(Set dst (MulReductionVF src1 src2));
19293 effect(TEMP dst);
19294 format %{ "vector_reduction_float $dst,$src1,$src2 ;" %}
19295 ins_encode %{
19296 int opcode = this->ideal_Opcode();
19297 int vlen = Matcher::vector_length(this, $src2);
19298 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister);
19299 %}
19300 ins_pipe( pipe_slow );
19301 %}
19302
19303 instruct unordered_reduction4F(regF dst, regF src1, vec src2, vec vtmp) %{
19304 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
19305 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19306 // src1 contains reduction identity
19307 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 4); // src2
19308 match(Set dst (AddReductionVF src1 src2));
19309 match(Set dst (MulReductionVF src1 src2));
19310 effect(TEMP dst, TEMP vtmp);
19311 format %{ "vector_reduction_float $dst,$src1,$src2 ; using $vtmp as TEMP" %}
19312 ins_encode %{
19313 int opcode = this->ideal_Opcode();
19314 int vlen = Matcher::vector_length(this, $src2);
19315 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp$$XMMRegister);
19316 %}
19317 ins_pipe( pipe_slow );
19318 %}
19319
19320 instruct unordered_reduction8F(regF dst, regF src1, vec src2, vec vtmp1, vec vtmp2) %{
19321 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
19322 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19323 // src1 contains reduction identity
19324 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src2
19325 match(Set dst (AddReductionVF src1 src2));
19326 match(Set dst (MulReductionVF src1 src2));
19327 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19328 format %{ "vector_reduction_float $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19329 ins_encode %{
19330 int opcode = this->ideal_Opcode();
19331 int vlen = Matcher::vector_length(this, $src2);
19332 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19333 %}
19334 ins_pipe( pipe_slow );
19335 %}
19336
19337 instruct unordered_reduction16F(regF dst, regF src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19338 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
19339 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19340 // src1 contains reduction identity
19341 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 16); // src2
19342 match(Set dst (AddReductionVF src1 src2));
19343 match(Set dst (MulReductionVF src1 src2));
19344 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19345 format %{ "vector_reduction_float $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19346 ins_encode %{
19347 int opcode = this->ideal_Opcode();
19348 int vlen = Matcher::vector_length(this, $src2);
19349 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19350 %}
19351 ins_pipe( pipe_slow );
19352 %}
19353
19354 // =======================Double Reduction==========================================
19355
19356 instruct reduction2D(regD dst, vec src, vec vtmp) %{
19357 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 2); // src
19358 match(Set dst (AddReductionVD dst src));
19359 match(Set dst (MulReductionVD dst src));
19360 effect(TEMP dst, TEMP vtmp);
19361 format %{ "vector_reduction_double $dst,$src ; using $vtmp as TEMP" %}
19362 ins_encode %{
19363 int opcode = this->ideal_Opcode();
19364 int vlen = Matcher::vector_length(this, $src);
19365 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister);
19366 %}
19367 ins_pipe( pipe_slow );
19368 %}
19369
19370 instruct reduction4D(regD dst, vec src, vec vtmp1, vec vtmp2) %{
19371 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 4); // src
19372 match(Set dst (AddReductionVD dst src));
19373 match(Set dst (MulReductionVD dst src));
19374 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19375 format %{ "vector_reduction_double $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
19376 ins_encode %{
19377 int opcode = this->ideal_Opcode();
19378 int vlen = Matcher::vector_length(this, $src);
19379 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19380 %}
19381 ins_pipe( pipe_slow );
19382 %}
19383
19384 instruct reduction8D(regD dst, legVec src, legVec vtmp1, legVec vtmp2) %{
19385 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src
19386 match(Set dst (AddReductionVD dst src));
19387 match(Set dst (MulReductionVD dst src));
19388 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19389 format %{ "vector_reduction_double $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
19390 ins_encode %{
19391 int opcode = this->ideal_Opcode();
19392 int vlen = Matcher::vector_length(this, $src);
19393 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19394 %}
19395 ins_pipe( pipe_slow );
19396 %}
19397
19398 instruct unordered_reduction2D(regD dst, regD src1, vec src2) %{
19399 // Non-strictly ordered floating-point add/mul reduction for doubles. This rule is
19400 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19401 // src1 contains reduction identity
19402 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 2); // src2
19403 match(Set dst (AddReductionVD src1 src2));
19404 match(Set dst (MulReductionVD src1 src2));
19405 effect(TEMP dst);
19406 format %{ "vector_reduction_double $dst,$src1,$src2 ;" %}
19407 ins_encode %{
19408 int opcode = this->ideal_Opcode();
19409 int vlen = Matcher::vector_length(this, $src2);
19410 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister);
19411 %}
19412 ins_pipe( pipe_slow );
19413 %}
19414
19415 instruct unordered_reduction4D(regD dst, regD src1, vec src2, vec vtmp) %{
19416 // Non-strictly ordered floating-point add/mul reduction for doubles. This rule is
19417 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19418 // src1 contains reduction identity
19419 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 4); // src2
19420 match(Set dst (AddReductionVD src1 src2));
19421 match(Set dst (MulReductionVD src1 src2));
19422 effect(TEMP dst, TEMP vtmp);
19423 format %{ "vector_reduction_double $dst,$src1,$src2 ; using $vtmp as TEMP" %}
19424 ins_encode %{
19425 int opcode = this->ideal_Opcode();
19426 int vlen = Matcher::vector_length(this, $src2);
19427 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp$$XMMRegister);
19428 %}
19429 ins_pipe( pipe_slow );
19430 %}
19431
19432 instruct unordered_reduction8D(regD dst, regD src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19433 // Non-strictly ordered floating-point add/mul reduction for doubles. This rule is
19434 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19435 // src1 contains reduction identity
19436 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src2
19437 match(Set dst (AddReductionVD src1 src2));
19438 match(Set dst (MulReductionVD src1 src2));
19439 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19440 format %{ "vector_reduction_double $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19441 ins_encode %{
19442 int opcode = this->ideal_Opcode();
19443 int vlen = Matcher::vector_length(this, $src2);
19444 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19445 %}
19446 ins_pipe( pipe_slow );
19447 %}
19448
19449 // =======================Byte Reduction==========================================
19450
19451 instruct reductionB(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19452 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE && !VM_Version::supports_avx512bw());
19453 match(Set dst (AddReductionVI src1 src2));
19454 match(Set dst (AndReductionV src1 src2));
19455 match(Set dst ( OrReductionV src1 src2));
19456 match(Set dst (XorReductionV src1 src2));
19457 match(Set dst (MinReductionV src1 src2));
19458 match(Set dst (MaxReductionV src1 src2));
19459 match(Set dst (UMinReductionV src1 src2));
19460 match(Set dst (UMaxReductionV src1 src2));
19461 effect(TEMP vtmp1, TEMP vtmp2);
19462 format %{ "vector_reduction_byte $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19463 ins_encode %{
19464 int opcode = this->ideal_Opcode();
19465 int vlen = Matcher::vector_length(this, $src2);
19466 __ reduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19467 %}
19468 ins_pipe( pipe_slow );
19469 %}
19470
19471 instruct reductionB_avx512bw(rRegI dst, rRegI src1, vec src2, vec vtmp1, vec vtmp2) %{
19472 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE && VM_Version::supports_avx512bw());
19473 match(Set dst (AddReductionVI src1 src2));
19474 match(Set dst (AndReductionV src1 src2));
19475 match(Set dst ( OrReductionV src1 src2));
19476 match(Set dst (XorReductionV src1 src2));
19477 match(Set dst (MinReductionV src1 src2));
19478 match(Set dst (MaxReductionV src1 src2));
19479 match(Set dst (UMinReductionV src1 src2));
19480 match(Set dst (UMaxReductionV src1 src2));
19481 effect(TEMP vtmp1, TEMP vtmp2);
19482 format %{ "vector_reduction_byte $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19483 ins_encode %{
19484 int opcode = this->ideal_Opcode();
19485 int vlen = Matcher::vector_length(this, $src2);
19486 __ reduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19487 %}
19488 ins_pipe( pipe_slow );
19489 %}
19490
19491 // =======================Short Reduction==========================================
19492
19493 instruct reductionS(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19494 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_SHORT); // src2
19495 match(Set dst (AddReductionVI src1 src2));
19496 match(Set dst (MulReductionVI src1 src2));
19497 match(Set dst (AndReductionV src1 src2));
19498 match(Set dst ( OrReductionV src1 src2));
19499 match(Set dst (XorReductionV src1 src2));
19500 match(Set dst (MinReductionV src1 src2));
19501 match(Set dst (MaxReductionV src1 src2));
19502 match(Set dst (UMinReductionV src1 src2));
19503 match(Set dst (UMaxReductionV src1 src2));
19504 effect(TEMP vtmp1, TEMP vtmp2);
19505 format %{ "vector_reduction_short $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19506 ins_encode %{
19507 int opcode = this->ideal_Opcode();
19508 int vlen = Matcher::vector_length(this, $src2);
19509 __ reduceS(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19510 %}
19511 ins_pipe( pipe_slow );
19512 %}
19513
19514 // =======================Mul Reduction==========================================
19515
19516 instruct mul_reductionB(rRegI dst, rRegI src1, vec src2, vec vtmp1, vec vtmp2) %{
19517 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE &&
19518 Matcher::vector_length(n->in(2)) <= 32); // src2
19519 match(Set dst (MulReductionVI src1 src2));
19520 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19521 format %{ "vector_mul_reduction_byte $dst,$src1,$src2; using $vtmp1, $vtmp2 as TEMP" %}
19522 ins_encode %{
19523 int opcode = this->ideal_Opcode();
19524 int vlen = Matcher::vector_length(this, $src2);
19525 __ mulreduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19526 %}
19527 ins_pipe( pipe_slow );
19528 %}
19529
19530 instruct mul_reduction64B(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19531 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE &&
19532 Matcher::vector_length(n->in(2)) == 64); // src2
19533 match(Set dst (MulReductionVI src1 src2));
19534 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19535 format %{ "vector_mul_reduction_byte $dst,$src1,$src2; using $vtmp1, $vtmp2 as TEMP" %}
19536 ins_encode %{
19537 int opcode = this->ideal_Opcode();
19538 int vlen = Matcher::vector_length(this, $src2);
19539 __ mulreduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19540 %}
19541 ins_pipe( pipe_slow );
19542 %}
19543
19544 //--------------------Min/Max Float Reduction --------------------
19545 // Float Min Reduction
19546 instruct minmax_reduction2F(legRegF dst, immF src1, legVec src2, legVec tmp, legVec atmp,
19547 legVec btmp, legVec xmm_1, rFlagsReg cr) %{
19548 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19549 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19550 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19551 Matcher::vector_length(n->in(2)) == 2);
19552 match(Set dst (MinReductionV src1 src2));
19553 match(Set dst (MaxReductionV src1 src2));
19554 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_1, KILL cr);
19555 format %{ "vector_minmax2F_reduction $dst,$src1,$src2 ; using $tmp, $atmp, $btmp, $xmm_1 as TEMP" %}
19556 ins_encode %{
19557 assert(UseAVX > 0, "sanity");
19558
19559 int opcode = this->ideal_Opcode();
19560 int vlen = Matcher::vector_length(this, $src2);
19561 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, $tmp$$XMMRegister,
19562 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_1$$XMMRegister);
19563 %}
19564 ins_pipe( pipe_slow );
19565 %}
19566
19567 instruct minmax_reductionF(legRegF dst, immF src1, legVec src2, legVec tmp, legVec atmp,
19568 legVec btmp, legVec xmm_0, legVec xmm_1, rFlagsReg cr) %{
19569 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19570 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19571 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19572 Matcher::vector_length(n->in(2)) >= 4);
19573 match(Set dst (MinReductionV src1 src2));
19574 match(Set dst (MaxReductionV src1 src2));
19575 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_0, TEMP xmm_1, KILL cr);
19576 format %{ "vector_minmaxF_reduction $dst,$src1,$src2 ; using $tmp, $atmp, $btmp, $xmm_0, $xmm_1 as TEMP" %}
19577 ins_encode %{
19578 assert(UseAVX > 0, "sanity");
19579
19580 int opcode = this->ideal_Opcode();
19581 int vlen = Matcher::vector_length(this, $src2);
19582 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, $tmp$$XMMRegister,
19583 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_0$$XMMRegister, $xmm_1$$XMMRegister);
19584 %}
19585 ins_pipe( pipe_slow );
19586 %}
19587
19588 instruct minmax_reduction2F_av(legRegF dst, legVec src, legVec tmp, legVec atmp,
19589 legVec btmp, legVec xmm_1, rFlagsReg cr) %{
19590 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19591 Matcher::vector_length(n->in(2)) == 2);
19592 match(Set dst (MinReductionV dst src));
19593 match(Set dst (MaxReductionV dst src));
19594 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_1, KILL cr);
19595 format %{ "vector_minmax2F_reduction $dst,$src ; using $tmp, $atmp, $btmp, $xmm_1 as TEMP" %}
19596 ins_encode %{
19597 assert(UseAVX > 0, "sanity");
19598
19599 int opcode = this->ideal_Opcode();
19600 int vlen = Matcher::vector_length(this, $src);
19601 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, $tmp$$XMMRegister,
19602 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_1$$XMMRegister);
19603 %}
19604 ins_pipe( pipe_slow );
19605 %}
19606
19607
19608 instruct minmax_reductionF_av(legRegF dst, legVec src, legVec tmp, legVec atmp, legVec btmp,
19609 legVec xmm_0, legVec xmm_1, rFlagsReg cr) %{
19610 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19611 Matcher::vector_length(n->in(2)) >= 4);
19612 match(Set dst (MinReductionV dst src));
19613 match(Set dst (MaxReductionV dst src));
19614 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_0, TEMP xmm_1, KILL cr);
19615 format %{ "vector_minmaxF_reduction $dst,$src ; using $tmp, $atmp, $btmp, $xmm_0, $xmm_1 as TEMP" %}
19616 ins_encode %{
19617 assert(UseAVX > 0, "sanity");
19618
19619 int opcode = this->ideal_Opcode();
19620 int vlen = Matcher::vector_length(this, $src);
19621 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, $tmp$$XMMRegister,
19622 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_0$$XMMRegister, $xmm_1$$XMMRegister);
19623 %}
19624 ins_pipe( pipe_slow );
19625 %}
19626
19627 instruct minmax_reduction2F_avx10_2(regF dst, immF src1, vec src2, vec xtmp1) %{
19628 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19629 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19630 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19631 Matcher::vector_length(n->in(2)) == 2);
19632 match(Set dst (MinReductionV src1 src2));
19633 match(Set dst (MaxReductionV src1 src2));
19634 effect(TEMP dst, TEMP xtmp1);
19635 format %{ "vector_minmax_reduction $dst, $src1, $src2 \t; using $xtmp1 as TEMP" %}
19636 ins_encode %{
19637 int opcode = this->ideal_Opcode();
19638 int vlen = Matcher::vector_length(this, $src2);
19639 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister,
19640 xnoreg, xnoreg, xnoreg, $xtmp1$$XMMRegister);
19641 %}
19642 ins_pipe( pipe_slow );
19643 %}
19644
19645 instruct minmax_reductionF_avx10_2(regF dst, immF src1, vec src2, vec xtmp1, vec xtmp2) %{
19646 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19647 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19648 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19649 Matcher::vector_length(n->in(2)) >= 4);
19650 match(Set dst (MinReductionV src1 src2));
19651 match(Set dst (MaxReductionV src1 src2));
19652 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19653 format %{ "vector_minmax_reduction $dst, $src1, $src2 \t; using $xtmp1 and $xtmp2 as TEMP" %}
19654 ins_encode %{
19655 int opcode = this->ideal_Opcode();
19656 int vlen = Matcher::vector_length(this, $src2);
19657 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, xnoreg, xnoreg,
19658 xnoreg, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19659 %}
19660 ins_pipe( pipe_slow );
19661 %}
19662
19663 instruct minmax_reduction2F_av_avx10_2(regF dst, vec src, vec xtmp1) %{
19664 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19665 Matcher::vector_length(n->in(2)) == 2);
19666 match(Set dst (MinReductionV dst src));
19667 match(Set dst (MaxReductionV dst src));
19668 effect(TEMP dst, TEMP xtmp1);
19669 format %{ "vector_minmax2F_reduction $dst, $src \t; using $xtmp1 as TEMP" %}
19670 ins_encode %{
19671 int opcode = this->ideal_Opcode();
19672 int vlen = Matcher::vector_length(this, $src);
19673 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, xnoreg, xnoreg, xnoreg,
19674 $xtmp1$$XMMRegister);
19675 %}
19676 ins_pipe( pipe_slow );
19677 %}
19678
19679 instruct minmax_reductionF_av_avx10_2(regF dst, vec src, vec xtmp1, vec xtmp2) %{
19680 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19681 Matcher::vector_length(n->in(2)) >= 4);
19682 match(Set dst (MinReductionV dst src));
19683 match(Set dst (MaxReductionV dst src));
19684 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19685 format %{ "vector_minmax2F_reduction $dst, $src \t; using $xtmp1 and $xtmp2 as TEMP" %}
19686 ins_encode %{
19687 int opcode = this->ideal_Opcode();
19688 int vlen = Matcher::vector_length(this, $src);
19689 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, xnoreg, xnoreg, xnoreg,
19690 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19691 %}
19692 ins_pipe( pipe_slow );
19693 %}
19694
19695 //--------------------Min Double Reduction --------------------
19696 instruct minmax_reduction2D(legRegD dst, immD src1, legVec src2, legVec tmp1, legVec tmp2,
19697 legVec tmp3, legVec tmp4, rFlagsReg cr) %{
19698 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19699 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19700 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19701 Matcher::vector_length(n->in(2)) == 2);
19702 match(Set dst (MinReductionV src1 src2));
19703 match(Set dst (MaxReductionV src1 src2));
19704 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
19705 format %{ "vector_minmax2D_reduction $dst,$src1,$src2 ; using $tmp1, $tmp2, $tmp3, $tmp4 as TEMP" %}
19706 ins_encode %{
19707 assert(UseAVX > 0, "sanity");
19708
19709 int opcode = this->ideal_Opcode();
19710 int vlen = Matcher::vector_length(this, $src2);
19711 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister,
19712 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister);
19713 %}
19714 ins_pipe( pipe_slow );
19715 %}
19716
19717 instruct minmax_reductionD(legRegD dst, immD src1, legVec src2, legVec tmp1, legVec tmp2,
19718 legVec tmp3, legVec tmp4, legVec tmp5, rFlagsReg cr) %{
19719 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19720 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19721 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19722 Matcher::vector_length(n->in(2)) >= 4);
19723 match(Set dst (MinReductionV src1 src2));
19724 match(Set dst (MaxReductionV src1 src2));
19725 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, KILL cr);
19726 format %{ "vector_minmaxD_reduction $dst,$src1,$src2 ; using $tmp1, $tmp2, $tmp3, $tmp4, $tmp5 as TEMP" %}
19727 ins_encode %{
19728 assert(UseAVX > 0, "sanity");
19729
19730 int opcode = this->ideal_Opcode();
19731 int vlen = Matcher::vector_length(this, $src2);
19732 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister,
19733 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister, $tmp5$$XMMRegister);
19734 %}
19735 ins_pipe( pipe_slow );
19736 %}
19737
19738
19739 instruct minmax_reduction2D_av(legRegD dst, legVec src, legVec tmp1, legVec tmp2,
19740 legVec tmp3, legVec tmp4, rFlagsReg cr) %{
19741 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19742 Matcher::vector_length(n->in(2)) == 2);
19743 match(Set dst (MinReductionV dst src));
19744 match(Set dst (MaxReductionV dst src));
19745 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
19746 format %{ "vector_minmax2D_reduction $dst,$src ; using $tmp1, $tmp2, $tmp3, $tmp4 as TEMP" %}
19747 ins_encode %{
19748 assert(UseAVX > 0, "sanity");
19749
19750 int opcode = this->ideal_Opcode();
19751 int vlen = Matcher::vector_length(this, $src);
19752 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19753 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister);
19754 %}
19755 ins_pipe( pipe_slow );
19756 %}
19757
19758 instruct minmax_reductionD_av(legRegD dst, legVec src, legVec tmp1, legVec tmp2, legVec tmp3,
19759 legVec tmp4, legVec tmp5, rFlagsReg cr) %{
19760 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19761 Matcher::vector_length(n->in(2)) >= 4);
19762 match(Set dst (MinReductionV dst src));
19763 match(Set dst (MaxReductionV dst src));
19764 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, KILL cr);
19765 format %{ "vector_minmaxD_reduction $dst,$src ; using $tmp1, $tmp2, $tmp3, $tmp4, $tmp5 as TEMP" %}
19766 ins_encode %{
19767 assert(UseAVX > 0, "sanity");
19768
19769 int opcode = this->ideal_Opcode();
19770 int vlen = Matcher::vector_length(this, $src);
19771 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19772 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister, $tmp5$$XMMRegister);
19773 %}
19774 ins_pipe( pipe_slow );
19775 %}
19776
19777 instruct minmax_reduction2D_avx10_2(regD dst, immD src1, vec src2, vec xtmp1) %{
19778 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19779 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19780 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19781 Matcher::vector_length(n->in(2)) == 2);
19782 match(Set dst (MinReductionV src1 src2));
19783 match(Set dst (MaxReductionV src1 src2));
19784 effect(TEMP dst, TEMP xtmp1);
19785 format %{ "vector_minmax2D_reduction $dst, $src1, $src2 ; using $xtmp1 as TEMP" %}
19786 ins_encode %{
19787 int opcode = this->ideal_Opcode();
19788 int vlen = Matcher::vector_length(this, $src2);
19789 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, xnoreg,
19790 xnoreg, xnoreg, $xtmp1$$XMMRegister);
19791 %}
19792 ins_pipe( pipe_slow );
19793 %}
19794
19795 instruct minmax_reductionD_avx10_2(regD dst, immD src1, vec src2, vec xtmp1, vec xtmp2) %{
19796 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19797 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19798 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19799 Matcher::vector_length(n->in(2)) >= 4);
19800 match(Set dst (MinReductionV src1 src2));
19801 match(Set dst (MaxReductionV src1 src2));
19802 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19803 format %{ "vector_minmaxD_reduction $dst, $src1, $src2 ; using $xtmp1 and $xtmp2 as TEMP" %}
19804 ins_encode %{
19805 int opcode = this->ideal_Opcode();
19806 int vlen = Matcher::vector_length(this, $src2);
19807 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, xnoreg, xnoreg,
19808 xnoreg, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19809 %}
19810 ins_pipe( pipe_slow );
19811 %}
19812
19813
19814 instruct minmax_reduction2D_av_avx10_2(regD dst, vec src, vec xtmp1) %{
19815 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19816 Matcher::vector_length(n->in(2)) == 2);
19817 match(Set dst (MinReductionV dst src));
19818 match(Set dst (MaxReductionV dst src));
19819 effect(TEMP dst, TEMP xtmp1);
19820 format %{ "vector_minmax2D_reduction $dst, $src ; using $xtmp1 as TEMP" %}
19821 ins_encode %{
19822 int opcode = this->ideal_Opcode();
19823 int vlen = Matcher::vector_length(this, $src);
19824 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19825 xnoreg, xnoreg, xnoreg, $xtmp1$$XMMRegister);
19826 %}
19827 ins_pipe( pipe_slow );
19828 %}
19829
19830 instruct minmax_reductionD_av_avx10_2(regD dst, vec src, vec xtmp1, vec xtmp2) %{
19831 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19832 Matcher::vector_length(n->in(2)) >= 4);
19833 match(Set dst (MinReductionV dst src));
19834 match(Set dst (MaxReductionV dst src));
19835 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19836 format %{ "vector_minmaxD_reduction $dst, $src ; using $xtmp1 and $xtmp2 as TEMP" %}
19837 ins_encode %{
19838 int opcode = this->ideal_Opcode();
19839 int vlen = Matcher::vector_length(this, $src);
19840 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19841 xnoreg, xnoreg, xnoreg, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19842 %}
19843 ins_pipe( pipe_slow );
19844 %}
19845
19846 // ====================VECTOR ARITHMETIC=======================================
19847
19848 // --------------------------------- ADD --------------------------------------
19849
19850 // Bytes vector add
19851 instruct vaddB(vec dst, vec src) %{
19852 predicate(UseAVX == 0);
19853 match(Set dst (AddVB dst src));
19854 format %{ "paddb $dst,$src\t! add packedB" %}
19855 ins_encode %{
19856 __ paddb($dst$$XMMRegister, $src$$XMMRegister);
19857 %}
19858 ins_pipe( pipe_slow );
19859 %}
19860
19861 instruct vaddB_reg(vec dst, vec src1, vec src2) %{
19862 predicate(UseAVX > 0);
19863 match(Set dst (AddVB src1 src2));
19864 format %{ "vpaddb $dst,$src1,$src2\t! add packedB" %}
19865 ins_encode %{
19866 int vlen_enc = vector_length_encoding(this);
19867 __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19868 %}
19869 ins_pipe( pipe_slow );
19870 %}
19871
19872 instruct vaddB_mem(vec dst, vec src, memory mem) %{
19873 predicate((UseAVX > 0) &&
19874 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19875 match(Set dst (AddVB src (LoadVector mem)));
19876 format %{ "vpaddb $dst,$src,$mem\t! add packedB" %}
19877 ins_encode %{
19878 int vlen_enc = vector_length_encoding(this);
19879 __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19880 %}
19881 ins_pipe( pipe_slow );
19882 %}
19883
19884 // Shorts/Chars vector add
19885 instruct vaddS(vec dst, vec src) %{
19886 predicate(UseAVX == 0);
19887 match(Set dst (AddVS dst src));
19888 format %{ "paddw $dst,$src\t! add packedS" %}
19889 ins_encode %{
19890 __ paddw($dst$$XMMRegister, $src$$XMMRegister);
19891 %}
19892 ins_pipe( pipe_slow );
19893 %}
19894
19895 instruct vaddS_reg(vec dst, vec src1, vec src2) %{
19896 predicate(UseAVX > 0);
19897 match(Set dst (AddVS src1 src2));
19898 format %{ "vpaddw $dst,$src1,$src2\t! add packedS" %}
19899 ins_encode %{
19900 int vlen_enc = vector_length_encoding(this);
19901 __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19902 %}
19903 ins_pipe( pipe_slow );
19904 %}
19905
19906 instruct vaddS_mem(vec dst, vec src, memory mem) %{
19907 predicate((UseAVX > 0) &&
19908 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19909 match(Set dst (AddVS src (LoadVector mem)));
19910 format %{ "vpaddw $dst,$src,$mem\t! add packedS" %}
19911 ins_encode %{
19912 int vlen_enc = vector_length_encoding(this);
19913 __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19914 %}
19915 ins_pipe( pipe_slow );
19916 %}
19917
19918 // Integers vector add
19919 instruct vaddI(vec dst, vec src) %{
19920 predicate(UseAVX == 0);
19921 match(Set dst (AddVI dst src));
19922 format %{ "paddd $dst,$src\t! add packedI" %}
19923 ins_encode %{
19924 __ paddd($dst$$XMMRegister, $src$$XMMRegister);
19925 %}
19926 ins_pipe( pipe_slow );
19927 %}
19928
19929 instruct vaddI_reg(vec dst, vec src1, vec src2) %{
19930 predicate(UseAVX > 0);
19931 match(Set dst (AddVI src1 src2));
19932 format %{ "vpaddd $dst,$src1,$src2\t! add packedI" %}
19933 ins_encode %{
19934 int vlen_enc = vector_length_encoding(this);
19935 __ vpaddd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19936 %}
19937 ins_pipe( pipe_slow );
19938 %}
19939
19940
19941 instruct vaddI_mem(vec dst, vec src, memory mem) %{
19942 predicate((UseAVX > 0) &&
19943 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19944 match(Set dst (AddVI src (LoadVector mem)));
19945 format %{ "vpaddd $dst,$src,$mem\t! add packedI" %}
19946 ins_encode %{
19947 int vlen_enc = vector_length_encoding(this);
19948 __ vpaddd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19949 %}
19950 ins_pipe( pipe_slow );
19951 %}
19952
19953 // Longs vector add
19954 instruct vaddL(vec dst, vec src) %{
19955 predicate(UseAVX == 0);
19956 match(Set dst (AddVL dst src));
19957 format %{ "paddq $dst,$src\t! add packedL" %}
19958 ins_encode %{
19959 __ paddq($dst$$XMMRegister, $src$$XMMRegister);
19960 %}
19961 ins_pipe( pipe_slow );
19962 %}
19963
19964 instruct vaddL_reg(vec dst, vec src1, vec src2) %{
19965 predicate(UseAVX > 0);
19966 match(Set dst (AddVL src1 src2));
19967 format %{ "vpaddq $dst,$src1,$src2\t! add packedL" %}
19968 ins_encode %{
19969 int vlen_enc = vector_length_encoding(this);
19970 __ vpaddq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19971 %}
19972 ins_pipe( pipe_slow );
19973 %}
19974
19975 instruct vaddL_mem(vec dst, vec src, memory mem) %{
19976 predicate((UseAVX > 0) &&
19977 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19978 match(Set dst (AddVL src (LoadVector mem)));
19979 format %{ "vpaddq $dst,$src,$mem\t! add packedL" %}
19980 ins_encode %{
19981 int vlen_enc = vector_length_encoding(this);
19982 __ vpaddq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19983 %}
19984 ins_pipe( pipe_slow );
19985 %}
19986
19987 // Floats vector add
19988 instruct vaddF(vec dst, vec src) %{
19989 predicate(UseAVX == 0);
19990 match(Set dst (AddVF dst src));
19991 format %{ "addps $dst,$src\t! add packedF" %}
19992 ins_encode %{
19993 __ addps($dst$$XMMRegister, $src$$XMMRegister);
19994 %}
19995 ins_pipe( pipe_slow );
19996 %}
19997
19998 instruct vaddF_reg(vec dst, vec src1, vec src2) %{
19999 predicate(UseAVX > 0);
20000 match(Set dst (AddVF src1 src2));
20001 format %{ "vaddps $dst,$src1,$src2\t! add packedF" %}
20002 ins_encode %{
20003 int vlen_enc = vector_length_encoding(this);
20004 __ vaddps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20005 %}
20006 ins_pipe( pipe_slow );
20007 %}
20008
20009 instruct vaddF_mem(vec dst, vec src, memory mem) %{
20010 predicate((UseAVX > 0) &&
20011 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20012 match(Set dst (AddVF src (LoadVector mem)));
20013 format %{ "vaddps $dst,$src,$mem\t! add packedF" %}
20014 ins_encode %{
20015 int vlen_enc = vector_length_encoding(this);
20016 __ vaddps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20017 %}
20018 ins_pipe( pipe_slow );
20019 %}
20020
20021 // Doubles vector add
20022 instruct vaddD(vec dst, vec src) %{
20023 predicate(UseAVX == 0);
20024 match(Set dst (AddVD dst src));
20025 format %{ "addpd $dst,$src\t! add packedD" %}
20026 ins_encode %{
20027 __ addpd($dst$$XMMRegister, $src$$XMMRegister);
20028 %}
20029 ins_pipe( pipe_slow );
20030 %}
20031
20032 instruct vaddD_reg(vec dst, vec src1, vec src2) %{
20033 predicate(UseAVX > 0);
20034 match(Set dst (AddVD src1 src2));
20035 format %{ "vaddpd $dst,$src1,$src2\t! add packedD" %}
20036 ins_encode %{
20037 int vlen_enc = vector_length_encoding(this);
20038 __ vaddpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20039 %}
20040 ins_pipe( pipe_slow );
20041 %}
20042
20043 instruct vaddD_mem(vec dst, vec src, memory mem) %{
20044 predicate((UseAVX > 0) &&
20045 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20046 match(Set dst (AddVD src (LoadVector mem)));
20047 format %{ "vaddpd $dst,$src,$mem\t! add packedD" %}
20048 ins_encode %{
20049 int vlen_enc = vector_length_encoding(this);
20050 __ vaddpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20051 %}
20052 ins_pipe( pipe_slow );
20053 %}
20054
20055 // --------------------------------- SUB --------------------------------------
20056
20057 // Bytes vector sub
20058 instruct vsubB(vec dst, vec src) %{
20059 predicate(UseAVX == 0);
20060 match(Set dst (SubVB dst src));
20061 format %{ "psubb $dst,$src\t! sub packedB" %}
20062 ins_encode %{
20063 __ psubb($dst$$XMMRegister, $src$$XMMRegister);
20064 %}
20065 ins_pipe( pipe_slow );
20066 %}
20067
20068 instruct vsubB_reg(vec dst, vec src1, vec src2) %{
20069 predicate(UseAVX > 0);
20070 match(Set dst (SubVB src1 src2));
20071 format %{ "vpsubb $dst,$src1,$src2\t! sub packedB" %}
20072 ins_encode %{
20073 int vlen_enc = vector_length_encoding(this);
20074 __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20075 %}
20076 ins_pipe( pipe_slow );
20077 %}
20078
20079 instruct vsubB_mem(vec dst, vec src, memory mem) %{
20080 predicate((UseAVX > 0) &&
20081 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20082 match(Set dst (SubVB src (LoadVector mem)));
20083 format %{ "vpsubb $dst,$src,$mem\t! sub packedB" %}
20084 ins_encode %{
20085 int vlen_enc = vector_length_encoding(this);
20086 __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20087 %}
20088 ins_pipe( pipe_slow );
20089 %}
20090
20091 // Shorts/Chars vector sub
20092 instruct vsubS(vec dst, vec src) %{
20093 predicate(UseAVX == 0);
20094 match(Set dst (SubVS dst src));
20095 format %{ "psubw $dst,$src\t! sub packedS" %}
20096 ins_encode %{
20097 __ psubw($dst$$XMMRegister, $src$$XMMRegister);
20098 %}
20099 ins_pipe( pipe_slow );
20100 %}
20101
20102
20103 instruct vsubS_reg(vec dst, vec src1, vec src2) %{
20104 predicate(UseAVX > 0);
20105 match(Set dst (SubVS src1 src2));
20106 format %{ "vpsubw $dst,$src1,$src2\t! sub packedS" %}
20107 ins_encode %{
20108 int vlen_enc = vector_length_encoding(this);
20109 __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20110 %}
20111 ins_pipe( pipe_slow );
20112 %}
20113
20114 instruct vsubS_mem(vec dst, vec src, memory mem) %{
20115 predicate((UseAVX > 0) &&
20116 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20117 match(Set dst (SubVS src (LoadVector mem)));
20118 format %{ "vpsubw $dst,$src,$mem\t! sub packedS" %}
20119 ins_encode %{
20120 int vlen_enc = vector_length_encoding(this);
20121 __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20122 %}
20123 ins_pipe( pipe_slow );
20124 %}
20125
20126 // Integers vector sub
20127 instruct vsubI(vec dst, vec src) %{
20128 predicate(UseAVX == 0);
20129 match(Set dst (SubVI dst src));
20130 format %{ "psubd $dst,$src\t! sub packedI" %}
20131 ins_encode %{
20132 __ psubd($dst$$XMMRegister, $src$$XMMRegister);
20133 %}
20134 ins_pipe( pipe_slow );
20135 %}
20136
20137 instruct vsubI_reg(vec dst, vec src1, vec src2) %{
20138 predicate(UseAVX > 0);
20139 match(Set dst (SubVI src1 src2));
20140 format %{ "vpsubd $dst,$src1,$src2\t! sub packedI" %}
20141 ins_encode %{
20142 int vlen_enc = vector_length_encoding(this);
20143 __ vpsubd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20144 %}
20145 ins_pipe( pipe_slow );
20146 %}
20147
20148 instruct vsubI_mem(vec dst, vec src, memory mem) %{
20149 predicate((UseAVX > 0) &&
20150 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20151 match(Set dst (SubVI src (LoadVector mem)));
20152 format %{ "vpsubd $dst,$src,$mem\t! sub packedI" %}
20153 ins_encode %{
20154 int vlen_enc = vector_length_encoding(this);
20155 __ vpsubd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20156 %}
20157 ins_pipe( pipe_slow );
20158 %}
20159
20160 // Longs vector sub
20161 instruct vsubL(vec dst, vec src) %{
20162 predicate(UseAVX == 0);
20163 match(Set dst (SubVL dst src));
20164 format %{ "psubq $dst,$src\t! sub packedL" %}
20165 ins_encode %{
20166 __ psubq($dst$$XMMRegister, $src$$XMMRegister);
20167 %}
20168 ins_pipe( pipe_slow );
20169 %}
20170
20171 instruct vsubL_reg(vec dst, vec src1, vec src2) %{
20172 predicate(UseAVX > 0);
20173 match(Set dst (SubVL src1 src2));
20174 format %{ "vpsubq $dst,$src1,$src2\t! sub packedL" %}
20175 ins_encode %{
20176 int vlen_enc = vector_length_encoding(this);
20177 __ vpsubq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20178 %}
20179 ins_pipe( pipe_slow );
20180 %}
20181
20182
20183 instruct vsubL_mem(vec dst, vec src, memory mem) %{
20184 predicate((UseAVX > 0) &&
20185 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20186 match(Set dst (SubVL src (LoadVector mem)));
20187 format %{ "vpsubq $dst,$src,$mem\t! sub packedL" %}
20188 ins_encode %{
20189 int vlen_enc = vector_length_encoding(this);
20190 __ vpsubq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20191 %}
20192 ins_pipe( pipe_slow );
20193 %}
20194
20195 // Floats vector sub
20196 instruct vsubF(vec dst, vec src) %{
20197 predicate(UseAVX == 0);
20198 match(Set dst (SubVF dst src));
20199 format %{ "subps $dst,$src\t! sub packedF" %}
20200 ins_encode %{
20201 __ subps($dst$$XMMRegister, $src$$XMMRegister);
20202 %}
20203 ins_pipe( pipe_slow );
20204 %}
20205
20206 instruct vsubF_reg(vec dst, vec src1, vec src2) %{
20207 predicate(UseAVX > 0);
20208 match(Set dst (SubVF src1 src2));
20209 format %{ "vsubps $dst,$src1,$src2\t! sub packedF" %}
20210 ins_encode %{
20211 int vlen_enc = vector_length_encoding(this);
20212 __ vsubps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20213 %}
20214 ins_pipe( pipe_slow );
20215 %}
20216
20217 instruct vsubF_mem(vec dst, vec src, memory mem) %{
20218 predicate((UseAVX > 0) &&
20219 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20220 match(Set dst (SubVF src (LoadVector mem)));
20221 format %{ "vsubps $dst,$src,$mem\t! sub packedF" %}
20222 ins_encode %{
20223 int vlen_enc = vector_length_encoding(this);
20224 __ vsubps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20225 %}
20226 ins_pipe( pipe_slow );
20227 %}
20228
20229 // Doubles vector sub
20230 instruct vsubD(vec dst, vec src) %{
20231 predicate(UseAVX == 0);
20232 match(Set dst (SubVD dst src));
20233 format %{ "subpd $dst,$src\t! sub packedD" %}
20234 ins_encode %{
20235 __ subpd($dst$$XMMRegister, $src$$XMMRegister);
20236 %}
20237 ins_pipe( pipe_slow );
20238 %}
20239
20240 instruct vsubD_reg(vec dst, vec src1, vec src2) %{
20241 predicate(UseAVX > 0);
20242 match(Set dst (SubVD src1 src2));
20243 format %{ "vsubpd $dst,$src1,$src2\t! sub packedD" %}
20244 ins_encode %{
20245 int vlen_enc = vector_length_encoding(this);
20246 __ vsubpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20247 %}
20248 ins_pipe( pipe_slow );
20249 %}
20250
20251 instruct vsubD_mem(vec dst, vec src, memory mem) %{
20252 predicate((UseAVX > 0) &&
20253 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20254 match(Set dst (SubVD src (LoadVector mem)));
20255 format %{ "vsubpd $dst,$src,$mem\t! sub packedD" %}
20256 ins_encode %{
20257 int vlen_enc = vector_length_encoding(this);
20258 __ vsubpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20259 %}
20260 ins_pipe( pipe_slow );
20261 %}
20262
20263 // --------------------------------- MUL --------------------------------------
20264
20265 // Byte vector mul
20266 instruct vmul8B(vec dst, vec src1, vec src2, vec xtmp) %{
20267 predicate(Matcher::vector_length_in_bytes(n) <= 8);
20268 match(Set dst (MulVB src1 src2));
20269 effect(TEMP dst, TEMP xtmp);
20270 format %{ "mulVB $dst, $src1, $src2\t! using $xtmp as TEMP" %}
20271 ins_encode %{
20272 assert(UseSSE > 3, "required");
20273 __ pmovsxbw($dst$$XMMRegister, $src1$$XMMRegister);
20274 __ pmovsxbw($xtmp$$XMMRegister, $src2$$XMMRegister);
20275 __ pmullw($dst$$XMMRegister, $xtmp$$XMMRegister);
20276 __ psllw($dst$$XMMRegister, 8);
20277 __ psrlw($dst$$XMMRegister, 8);
20278 __ packuswb($dst$$XMMRegister, $dst$$XMMRegister);
20279 %}
20280 ins_pipe( pipe_slow );
20281 %}
20282
20283 instruct vmulB(vec dst, vec src1, vec src2, vec xtmp) %{
20284 predicate(UseAVX == 0 && Matcher::vector_length_in_bytes(n) > 8);
20285 match(Set dst (MulVB src1 src2));
20286 effect(TEMP dst, TEMP xtmp);
20287 format %{ "mulVB $dst, $src1, $src2\t! using $xtmp as TEMP" %}
20288 ins_encode %{
20289 assert(UseSSE > 3, "required");
20290 // Odd-index elements
20291 __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
20292 __ psrlw($dst$$XMMRegister, 8);
20293 __ movdqu($xtmp$$XMMRegister, $src2$$XMMRegister);
20294 __ psrlw($xtmp$$XMMRegister, 8);
20295 __ pmullw($dst$$XMMRegister, $xtmp$$XMMRegister);
20296 __ psllw($dst$$XMMRegister, 8);
20297 // Even-index elements
20298 __ movdqu($xtmp$$XMMRegister, $src1$$XMMRegister);
20299 __ pmullw($xtmp$$XMMRegister, $src2$$XMMRegister);
20300 __ psllw($xtmp$$XMMRegister, 8);
20301 __ psrlw($xtmp$$XMMRegister, 8);
20302 // Combine
20303 __ por($dst$$XMMRegister, $xtmp$$XMMRegister);
20304 %}
20305 ins_pipe( pipe_slow );
20306 %}
20307
20308 instruct vmulB_reg(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2) %{
20309 predicate(UseAVX > 0 && Matcher::vector_length_in_bytes(n) > 8);
20310 match(Set dst (MulVB src1 src2));
20311 effect(TEMP xtmp1, TEMP xtmp2);
20312 format %{ "vmulVB $dst, $src1, $src2\t! using $xtmp1, $xtmp2 as TEMP" %}
20313 ins_encode %{
20314 int vlen_enc = vector_length_encoding(this);
20315 // Odd-index elements
20316 __ vpsrlw($xtmp2$$XMMRegister, $src1$$XMMRegister, 8, vlen_enc);
20317 __ vpsrlw($xtmp1$$XMMRegister, $src2$$XMMRegister, 8, vlen_enc);
20318 __ vpmullw($xtmp2$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20319 __ vpsllw($xtmp2$$XMMRegister, $xtmp2$$XMMRegister, 8, vlen_enc);
20320 // Even-index elements
20321 __ vpmullw($xtmp1$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20322 __ vpsllw($xtmp1$$XMMRegister, $xtmp1$$XMMRegister, 8, vlen_enc);
20323 __ vpsrlw($xtmp1$$XMMRegister, $xtmp1$$XMMRegister, 8, vlen_enc);
20324 // Combine
20325 __ vpor($dst$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20326 %}
20327 ins_pipe( pipe_slow );
20328 %}
20329
20330 // Shorts/Chars vector mul
20331 instruct vmulS(vec dst, vec src) %{
20332 predicate(UseAVX == 0);
20333 match(Set dst (MulVS dst src));
20334 format %{ "pmullw $dst,$src\t! mul packedS" %}
20335 ins_encode %{
20336 __ pmullw($dst$$XMMRegister, $src$$XMMRegister);
20337 %}
20338 ins_pipe( pipe_slow );
20339 %}
20340
20341 instruct vmulS_reg(vec dst, vec src1, vec src2) %{
20342 predicate(UseAVX > 0);
20343 match(Set dst (MulVS src1 src2));
20344 format %{ "vpmullw $dst,$src1,$src2\t! mul packedS" %}
20345 ins_encode %{
20346 int vlen_enc = vector_length_encoding(this);
20347 __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20348 %}
20349 ins_pipe( pipe_slow );
20350 %}
20351
20352 instruct vmulS_mem(vec dst, vec src, memory mem) %{
20353 predicate((UseAVX > 0) &&
20354 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20355 match(Set dst (MulVS src (LoadVector mem)));
20356 format %{ "vpmullw $dst,$src,$mem\t! mul packedS" %}
20357 ins_encode %{
20358 int vlen_enc = vector_length_encoding(this);
20359 __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20360 %}
20361 ins_pipe( pipe_slow );
20362 %}
20363
20364 // Integers vector mul
20365 instruct vmulI(vec dst, vec src) %{
20366 predicate(UseAVX == 0);
20367 match(Set dst (MulVI dst src));
20368 format %{ "pmulld $dst,$src\t! mul packedI" %}
20369 ins_encode %{
20370 assert(UseSSE > 3, "required");
20371 __ pmulld($dst$$XMMRegister, $src$$XMMRegister);
20372 %}
20373 ins_pipe( pipe_slow );
20374 %}
20375
20376 instruct vmulI_reg(vec dst, vec src1, vec src2) %{
20377 predicate(UseAVX > 0);
20378 match(Set dst (MulVI src1 src2));
20379 format %{ "vpmulld $dst,$src1,$src2\t! mul packedI" %}
20380 ins_encode %{
20381 int vlen_enc = vector_length_encoding(this);
20382 __ vpmulld($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20383 %}
20384 ins_pipe( pipe_slow );
20385 %}
20386
20387 instruct vmulI_mem(vec dst, vec src, memory mem) %{
20388 predicate((UseAVX > 0) &&
20389 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20390 match(Set dst (MulVI src (LoadVector mem)));
20391 format %{ "vpmulld $dst,$src,$mem\t! mul packedI" %}
20392 ins_encode %{
20393 int vlen_enc = vector_length_encoding(this);
20394 __ vpmulld($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20395 %}
20396 ins_pipe( pipe_slow );
20397 %}
20398
20399 // Longs vector mul
20400 instruct evmulL_reg(vec dst, vec src1, vec src2) %{
20401 predicate((Matcher::vector_length_in_bytes(n) == 64 &&
20402 VM_Version::supports_avx512dq()) ||
20403 VM_Version::supports_avx512vldq());
20404 match(Set dst (MulVL src1 src2));
20405 ins_cost(500);
20406 format %{ "evpmullq $dst,$src1,$src2\t! mul packedL" %}
20407 ins_encode %{
20408 assert(UseAVX > 2, "required");
20409 int vlen_enc = vector_length_encoding(this);
20410 __ evpmullq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20411 %}
20412 ins_pipe( pipe_slow );
20413 %}
20414
20415 instruct evmulL_mem(vec dst, vec src, memory mem) %{
20416 predicate((Matcher::vector_length_in_bytes(n) == 64 &&
20417 VM_Version::supports_avx512dq()) ||
20418 (Matcher::vector_length_in_bytes(n) > 8 &&
20419 VM_Version::supports_avx512vldq()));
20420 match(Set dst (MulVL src (LoadVector mem)));
20421 format %{ "evpmullq $dst,$src,$mem\t! mul packedL" %}
20422 ins_cost(500);
20423 ins_encode %{
20424 assert(UseAVX > 2, "required");
20425 int vlen_enc = vector_length_encoding(this);
20426 __ evpmullq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20427 %}
20428 ins_pipe( pipe_slow );
20429 %}
20430
20431 instruct vmulL(vec dst, vec src1, vec src2, vec xtmp) %{
20432 predicate(UseAVX == 0);
20433 match(Set dst (MulVL src1 src2));
20434 ins_cost(500);
20435 effect(TEMP dst, TEMP xtmp);
20436 format %{ "mulVL $dst, $src1, $src2\t! using $xtmp as TEMP" %}
20437 ins_encode %{
20438 assert(VM_Version::supports_sse4_1(), "required");
20439 // Get the lo-hi products, only the lower 32 bits is in concerns
20440 __ pshufd($xtmp$$XMMRegister, $src2$$XMMRegister, 0xB1);
20441 __ pmulld($xtmp$$XMMRegister, $src1$$XMMRegister);
20442 __ pshufd($dst$$XMMRegister, $xtmp$$XMMRegister, 0xB1);
20443 __ paddd($dst$$XMMRegister, $xtmp$$XMMRegister);
20444 __ psllq($dst$$XMMRegister, 32);
20445 // Get the lo-lo products
20446 __ movdqu($xtmp$$XMMRegister, $src1$$XMMRegister);
20447 __ pmuludq($xtmp$$XMMRegister, $src2$$XMMRegister);
20448 __ paddq($dst$$XMMRegister, $xtmp$$XMMRegister);
20449 %}
20450 ins_pipe( pipe_slow );
20451 %}
20452
20453 instruct vmulL_reg(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2) %{
20454 predicate(UseAVX > 0 &&
20455 ((Matcher::vector_length_in_bytes(n) == 64 &&
20456 !VM_Version::supports_avx512dq()) ||
20457 (Matcher::vector_length_in_bytes(n) < 64 &&
20458 !VM_Version::supports_avx512vldq())));
20459 match(Set dst (MulVL src1 src2));
20460 effect(TEMP xtmp1, TEMP xtmp2);
20461 ins_cost(500);
20462 format %{ "vmulVL $dst, $src1, $src2\t! using $xtmp1, $xtmp2 as TEMP" %}
20463 ins_encode %{
20464 int vlen_enc = vector_length_encoding(this);
20465 // Get the lo-hi products, only the lower 32 bits is in concerns
20466 __ vpshufd($xtmp1$$XMMRegister, $src2$$XMMRegister, 0xB1, vlen_enc);
20467 __ vpmulld($xtmp1$$XMMRegister, $src1$$XMMRegister, $xtmp1$$XMMRegister, vlen_enc);
20468 __ vpshufd($xtmp2$$XMMRegister, $xtmp1$$XMMRegister, 0xB1, vlen_enc);
20469 __ vpaddd($xtmp2$$XMMRegister, $xtmp2$$XMMRegister, $xtmp1$$XMMRegister, vlen_enc);
20470 __ vpsllq($xtmp2$$XMMRegister, $xtmp2$$XMMRegister, 32, vlen_enc);
20471 // Get the lo-lo products
20472 __ vpmuludq($xtmp1$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20473 __ vpaddq($dst$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20474 %}
20475 ins_pipe( pipe_slow );
20476 %}
20477
20478 instruct vmuludq_reg(vec dst, vec src1, vec src2) %{
20479 predicate(UseAVX > 0 && n->as_MulVL()->has_uint_inputs());
20480 match(Set dst (MulVL src1 src2));
20481 ins_cost(100);
20482 format %{ "vpmuludq $dst,$src1,$src2\t! muludq packedL" %}
20483 ins_encode %{
20484 int vlen_enc = vector_length_encoding(this);
20485 __ vpmuludq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20486 %}
20487 ins_pipe( pipe_slow );
20488 %}
20489
20490 instruct vmuldq_reg(vec dst, vec src1, vec src2) %{
20491 predicate(UseAVX > 0 && n->as_MulVL()->has_int_inputs());
20492 match(Set dst (MulVL src1 src2));
20493 ins_cost(100);
20494 format %{ "vpmuldq $dst,$src1,$src2\t! muldq packedL" %}
20495 ins_encode %{
20496 int vlen_enc = vector_length_encoding(this);
20497 __ vpmuldq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20498 %}
20499 ins_pipe( pipe_slow );
20500 %}
20501
20502 // Floats vector mul
20503 instruct vmulF(vec dst, vec src) %{
20504 predicate(UseAVX == 0);
20505 match(Set dst (MulVF dst src));
20506 format %{ "mulps $dst,$src\t! mul packedF" %}
20507 ins_encode %{
20508 __ mulps($dst$$XMMRegister, $src$$XMMRegister);
20509 %}
20510 ins_pipe( pipe_slow );
20511 %}
20512
20513 instruct vmulF_reg(vec dst, vec src1, vec src2) %{
20514 predicate(UseAVX > 0);
20515 match(Set dst (MulVF src1 src2));
20516 format %{ "vmulps $dst,$src1,$src2\t! mul packedF" %}
20517 ins_encode %{
20518 int vlen_enc = vector_length_encoding(this);
20519 __ vmulps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20520 %}
20521 ins_pipe( pipe_slow );
20522 %}
20523
20524 instruct vmulF_mem(vec dst, vec src, memory mem) %{
20525 predicate((UseAVX > 0) &&
20526 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20527 match(Set dst (MulVF src (LoadVector mem)));
20528 format %{ "vmulps $dst,$src,$mem\t! mul packedF" %}
20529 ins_encode %{
20530 int vlen_enc = vector_length_encoding(this);
20531 __ vmulps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20532 %}
20533 ins_pipe( pipe_slow );
20534 %}
20535
20536 // Doubles vector mul
20537 instruct vmulD(vec dst, vec src) %{
20538 predicate(UseAVX == 0);
20539 match(Set dst (MulVD dst src));
20540 format %{ "mulpd $dst,$src\t! mul packedD" %}
20541 ins_encode %{
20542 __ mulpd($dst$$XMMRegister, $src$$XMMRegister);
20543 %}
20544 ins_pipe( pipe_slow );
20545 %}
20546
20547 instruct vmulD_reg(vec dst, vec src1, vec src2) %{
20548 predicate(UseAVX > 0);
20549 match(Set dst (MulVD src1 src2));
20550 format %{ "vmulpd $dst,$src1,$src2\t! mul packedD" %}
20551 ins_encode %{
20552 int vlen_enc = vector_length_encoding(this);
20553 __ vmulpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20554 %}
20555 ins_pipe( pipe_slow );
20556 %}
20557
20558 instruct vmulD_mem(vec dst, vec src, memory mem) %{
20559 predicate((UseAVX > 0) &&
20560 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20561 match(Set dst (MulVD src (LoadVector mem)));
20562 format %{ "vmulpd $dst,$src,$mem\t! mul packedD" %}
20563 ins_encode %{
20564 int vlen_enc = vector_length_encoding(this);
20565 __ vmulpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20566 %}
20567 ins_pipe( pipe_slow );
20568 %}
20569
20570 // --------------------------------- DIV --------------------------------------
20571
20572 // Floats vector div
20573 instruct vdivF(vec dst, vec src) %{
20574 predicate(UseAVX == 0);
20575 match(Set dst (DivVF dst src));
20576 format %{ "divps $dst,$src\t! div packedF" %}
20577 ins_encode %{
20578 __ divps($dst$$XMMRegister, $src$$XMMRegister);
20579 %}
20580 ins_pipe( pipe_slow );
20581 %}
20582
20583 instruct vdivF_reg(vec dst, vec src1, vec src2) %{
20584 predicate(UseAVX > 0);
20585 match(Set dst (DivVF src1 src2));
20586 format %{ "vdivps $dst,$src1,$src2\t! div packedF" %}
20587 ins_encode %{
20588 int vlen_enc = vector_length_encoding(this);
20589 __ vdivps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20590 %}
20591 ins_pipe( pipe_slow );
20592 %}
20593
20594 instruct vdivF_mem(vec dst, vec src, memory mem) %{
20595 predicate((UseAVX > 0) &&
20596 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20597 match(Set dst (DivVF src (LoadVector mem)));
20598 format %{ "vdivps $dst,$src,$mem\t! div packedF" %}
20599 ins_encode %{
20600 int vlen_enc = vector_length_encoding(this);
20601 __ vdivps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20602 %}
20603 ins_pipe( pipe_slow );
20604 %}
20605
20606 // Doubles vector div
20607 instruct vdivD(vec dst, vec src) %{
20608 predicate(UseAVX == 0);
20609 match(Set dst (DivVD dst src));
20610 format %{ "divpd $dst,$src\t! div packedD" %}
20611 ins_encode %{
20612 __ divpd($dst$$XMMRegister, $src$$XMMRegister);
20613 %}
20614 ins_pipe( pipe_slow );
20615 %}
20616
20617 instruct vdivD_reg(vec dst, vec src1, vec src2) %{
20618 predicate(UseAVX > 0);
20619 match(Set dst (DivVD src1 src2));
20620 format %{ "vdivpd $dst,$src1,$src2\t! div packedD" %}
20621 ins_encode %{
20622 int vlen_enc = vector_length_encoding(this);
20623 __ vdivpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20624 %}
20625 ins_pipe( pipe_slow );
20626 %}
20627
20628 instruct vdivD_mem(vec dst, vec src, memory mem) %{
20629 predicate((UseAVX > 0) &&
20630 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20631 match(Set dst (DivVD src (LoadVector mem)));
20632 format %{ "vdivpd $dst,$src,$mem\t! div packedD" %}
20633 ins_encode %{
20634 int vlen_enc = vector_length_encoding(this);
20635 __ vdivpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20636 %}
20637 ins_pipe( pipe_slow );
20638 %}
20639
20640 // ------------------------------ MinMax ---------------------------------------
20641
20642 // Byte, Short, Int vector Min/Max
20643 instruct minmax_reg_sse(vec dst, vec src) %{
20644 predicate(is_integral_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_element_basic_type(n) != T_LONG && // T_BYTE, T_SHORT, T_INT
20645 UseAVX == 0);
20646 match(Set dst (MinV dst src));
20647 match(Set dst (MaxV dst src));
20648 format %{ "vector_minmax $dst,$src\t! " %}
20649 ins_encode %{
20650 assert(UseSSE >= 4, "required");
20651
20652 int opcode = this->ideal_Opcode();
20653 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20654 __ pminmax(opcode, elem_bt, $dst$$XMMRegister, $src$$XMMRegister);
20655 %}
20656 ins_pipe( pipe_slow );
20657 %}
20658
20659 instruct vminmax_reg(vec dst, vec src1, vec src2) %{
20660 predicate(is_integral_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_element_basic_type(n) != T_LONG && // T_BYTE, T_SHORT, T_INT
20661 UseAVX > 0);
20662 match(Set dst (MinV src1 src2));
20663 match(Set dst (MaxV src1 src2));
20664 format %{ "vector_minmax $dst,$src1,$src2\t! " %}
20665 ins_encode %{
20666 int opcode = this->ideal_Opcode();
20667 int vlen_enc = vector_length_encoding(this);
20668 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20669
20670 __ vpminmax(opcode, elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20671 %}
20672 ins_pipe( pipe_slow );
20673 %}
20674
20675 // Long vector Min/Max
20676 instruct minmaxL_reg_sse(vec dst, vec src, rxmm0 tmp) %{
20677 predicate(Matcher::vector_length_in_bytes(n) == 16 && Matcher::vector_element_basic_type(n) == T_LONG &&
20678 UseAVX == 0);
20679 match(Set dst (MinV dst src));
20680 match(Set dst (MaxV src dst));
20681 effect(TEMP dst, TEMP tmp);
20682 format %{ "vector_minmaxL $dst,$src\t!using $tmp as TEMP" %}
20683 ins_encode %{
20684 assert(UseSSE >= 4, "required");
20685
20686 int opcode = this->ideal_Opcode();
20687 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20688 assert(elem_bt == T_LONG, "sanity");
20689
20690 __ pminmax(opcode, elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $tmp$$XMMRegister);
20691 %}
20692 ins_pipe( pipe_slow );
20693 %}
20694
20695 instruct vminmaxL_reg_avx(legVec dst, legVec src1, legVec src2) %{
20696 predicate(Matcher::vector_length_in_bytes(n) <= 32 && Matcher::vector_element_basic_type(n) == T_LONG &&
20697 UseAVX > 0 && !VM_Version::supports_avx512vl());
20698 match(Set dst (MinV src1 src2));
20699 match(Set dst (MaxV src1 src2));
20700 effect(TEMP dst);
20701 format %{ "vector_minmaxL $dst,$src1,$src2\t! " %}
20702 ins_encode %{
20703 int vlen_enc = vector_length_encoding(this);
20704 int opcode = this->ideal_Opcode();
20705 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20706 assert(elem_bt == T_LONG, "sanity");
20707
20708 __ vpminmax(opcode, elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20709 %}
20710 ins_pipe( pipe_slow );
20711 %}
20712
20713 instruct vminmaxL_reg_evex(vec dst, vec src1, vec src2) %{
20714 predicate((Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()) &&
20715 Matcher::vector_element_basic_type(n) == T_LONG);
20716 match(Set dst (MinV src1 src2));
20717 match(Set dst (MaxV src1 src2));
20718 format %{ "vector_minmaxL $dst,$src1,src2\t! " %}
20719 ins_encode %{
20720 assert(UseAVX > 2, "required");
20721
20722 int vlen_enc = vector_length_encoding(this);
20723 int opcode = this->ideal_Opcode();
20724 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20725 assert(elem_bt == T_LONG, "sanity");
20726
20727 __ vpminmax(opcode, elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20728 %}
20729 ins_pipe( pipe_slow );
20730 %}
20731
20732 // Float/Double vector Min/Max
20733 instruct minmaxFP_reg_avx10_2(vec dst, vec a, vec b) %{
20734 predicate(VM_Version::supports_avx10_2() &&
20735 is_floating_point_type(Matcher::vector_element_basic_type(n))); // T_FLOAT, T_DOUBLE
20736 match(Set dst (MinV a b));
20737 match(Set dst (MaxV a b));
20738 format %{ "vector_minmaxFP $dst, $a, $b" %}
20739 ins_encode %{
20740 int vlen_enc = vector_length_encoding(this);
20741 int opcode = this->ideal_Opcode();
20742 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20743 __ vminmax_fp_avx10_2(opcode, elem_bt, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister, vlen_enc);
20744 %}
20745 ins_pipe( pipe_slow );
20746 %}
20747
20748 // Float/Double vector Min/Max
20749 instruct minmaxFP_reg(legVec dst, legVec a, legVec b, legVec tmp, legVec atmp, legVec btmp) %{
20750 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_length_in_bytes(n) <= 32 &&
20751 is_floating_point_type(Matcher::vector_element_basic_type(n)) && // T_FLOAT, T_DOUBLE
20752 UseAVX > 0);
20753 match(Set dst (MinV a b));
20754 match(Set dst (MaxV a b));
20755 effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
20756 format %{ "vector_minmaxFP $dst,$a,$b\t!using $tmp, $atmp, $btmp as TEMP" %}
20757 ins_encode %{
20758 assert(UseAVX > 0, "required");
20759
20760 int opcode = this->ideal_Opcode();
20761 int vlen_enc = vector_length_encoding(this);
20762 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20763
20764 __ vminmax_fp(opcode, elem_bt,
20765 $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister,
20766 $tmp$$XMMRegister, $atmp$$XMMRegister , $btmp$$XMMRegister, vlen_enc);
20767 %}
20768 ins_pipe( pipe_slow );
20769 %}
20770
20771 instruct evminmaxFP_reg_evex(vec dst, vec a, vec b, vec atmp, vec btmp, kReg ktmp) %{
20772 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_length_in_bytes(n) == 64 &&
20773 is_floating_point_type(Matcher::vector_element_basic_type(n))); // T_FLOAT, T_DOUBLE
20774 match(Set dst (MinV a b));
20775 match(Set dst (MaxV a b));
20776 effect(TEMP dst, USE a, USE b, TEMP atmp, TEMP btmp, TEMP ktmp);
20777 format %{ "vector_minmaxFP $dst,$a,$b\t!using $atmp, $btmp as TEMP" %}
20778 ins_encode %{
20779 assert(UseAVX > 2, "required");
20780
20781 int opcode = this->ideal_Opcode();
20782 int vlen_enc = vector_length_encoding(this);
20783 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20784
20785 __ evminmax_fp(opcode, elem_bt,
20786 $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister,
20787 $ktmp$$KRegister, $atmp$$XMMRegister , $btmp$$XMMRegister, vlen_enc);
20788 %}
20789 ins_pipe( pipe_slow );
20790 %}
20791
20792 // ------------------------------ Unsigned vector Min/Max ----------------------
20793
20794 instruct vector_uminmax_reg(vec dst, vec a, vec b) %{
20795 predicate(VM_Version::supports_avx512vl() || Matcher::vector_element_basic_type(n) != T_LONG);
20796 match(Set dst (UMinV a b));
20797 match(Set dst (UMaxV a b));
20798 format %{ "vector_uminmax $dst,$a,$b\t!" %}
20799 ins_encode %{
20800 int opcode = this->ideal_Opcode();
20801 int vlen_enc = vector_length_encoding(this);
20802 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20803 assert(is_integral_type(elem_bt), "");
20804 __ vpuminmax(opcode, elem_bt, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vlen_enc);
20805 %}
20806 ins_pipe( pipe_slow );
20807 %}
20808
20809 instruct vector_uminmax_mem(vec dst, vec a, memory b) %{
20810 predicate(VM_Version::supports_avx512vl() || Matcher::vector_element_basic_type(n) != T_LONG);
20811 match(Set dst (UMinV a (LoadVector b)));
20812 match(Set dst (UMaxV a (LoadVector b)));
20813 format %{ "vector_uminmax $dst,$a,$b\t!" %}
20814 ins_encode %{
20815 int opcode = this->ideal_Opcode();
20816 int vlen_enc = vector_length_encoding(this);
20817 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20818 assert(is_integral_type(elem_bt), "");
20819 __ vpuminmax(opcode, elem_bt, $dst$$XMMRegister, $a$$XMMRegister, $b$$Address, vlen_enc);
20820 %}
20821 ins_pipe( pipe_slow );
20822 %}
20823
20824 instruct vector_uminmaxq_reg(vec dst, vec a, vec b, vec xtmp1, vec xtmp2) %{
20825 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_element_basic_type(n) == T_LONG);
20826 match(Set dst (UMinV a b));
20827 match(Set dst (UMaxV a b));
20828 effect(TEMP xtmp1, TEMP xtmp2);
20829 format %{ "vector_uminmaxq $dst,$a,$b\t! using xtmp1 and xtmp2 as TEMP" %}
20830 ins_encode %{
20831 int opcode = this->ideal_Opcode();
20832 int vlen_enc = vector_length_encoding(this);
20833 __ vpuminmaxq(opcode, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20834 %}
20835 ins_pipe( pipe_slow );
20836 %}
20837
20838 instruct vector_uminmax_reg_masked(vec dst, vec src2, kReg mask) %{
20839 match(Set dst (UMinV (Binary dst src2) mask));
20840 match(Set dst (UMaxV (Binary dst src2) mask));
20841 format %{ "vector_uminmax_masked $dst, $dst, $src2, $mask\t! umin/max masked operation" %}
20842 ins_encode %{
20843 int vlen_enc = vector_length_encoding(this);
20844 BasicType bt = Matcher::vector_element_basic_type(this);
20845 int opc = this->ideal_Opcode();
20846 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
20847 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
20848 %}
20849 ins_pipe( pipe_slow );
20850 %}
20851
20852 instruct vector_uminmax_mem_masked(vec dst, memory src2, kReg mask) %{
20853 match(Set dst (UMinV (Binary dst (LoadVector src2)) mask));
20854 match(Set dst (UMaxV (Binary dst (LoadVector src2)) mask));
20855 format %{ "vector_uminmax_masked $dst, $dst, $src2, $mask\t! umin/max masked operation" %}
20856 ins_encode %{
20857 int vlen_enc = vector_length_encoding(this);
20858 BasicType bt = Matcher::vector_element_basic_type(this);
20859 int opc = this->ideal_Opcode();
20860 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
20861 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
20862 %}
20863 ins_pipe( pipe_slow );
20864 %}
20865
20866 // --------------------------------- Signum/CopySign ---------------------------
20867
20868 instruct signumF_reg(regF dst, regF zero, regF one, rFlagsReg cr) %{
20869 match(Set dst (SignumF dst (Binary zero one)));
20870 effect(KILL cr);
20871 format %{ "signumF $dst, $dst" %}
20872 ins_encode %{
20873 int opcode = this->ideal_Opcode();
20874 __ signum_fp(opcode, $dst$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister);
20875 %}
20876 ins_pipe( pipe_slow );
20877 %}
20878
20879 instruct signumD_reg(regD dst, regD zero, regD one, rFlagsReg cr) %{
20880 match(Set dst (SignumD dst (Binary zero one)));
20881 effect(KILL cr);
20882 format %{ "signumD $dst, $dst" %}
20883 ins_encode %{
20884 int opcode = this->ideal_Opcode();
20885 __ signum_fp(opcode, $dst$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister);
20886 %}
20887 ins_pipe( pipe_slow );
20888 %}
20889
20890 instruct signumV_reg_avx(vec dst, vec src, vec zero, vec one, vec xtmp1) %{
20891 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n) <= 32);
20892 match(Set dst (SignumVF src (Binary zero one)));
20893 match(Set dst (SignumVD src (Binary zero one)));
20894 effect(TEMP dst, TEMP xtmp1);
20895 format %{ "vector_signum_avx $dst, $src\t! using $xtmp1 as TEMP" %}
20896 ins_encode %{
20897 int opcode = this->ideal_Opcode();
20898 int vec_enc = vector_length_encoding(this);
20899 __ vector_signum_avx(opcode, $dst$$XMMRegister, $src$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister,
20900 $xtmp1$$XMMRegister, vec_enc);
20901 %}
20902 ins_pipe( pipe_slow );
20903 %}
20904
20905 instruct signumV_reg_evex(vec dst, vec src, vec zero, vec one, kReg ktmp1) %{
20906 predicate(VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64);
20907 match(Set dst (SignumVF src (Binary zero one)));
20908 match(Set dst (SignumVD src (Binary zero one)));
20909 effect(TEMP dst, TEMP ktmp1);
20910 format %{ "vector_signum_evex $dst, $src\t! using $ktmp1 as TEMP" %}
20911 ins_encode %{
20912 int opcode = this->ideal_Opcode();
20913 int vec_enc = vector_length_encoding(this);
20914 __ vector_signum_evex(opcode, $dst$$XMMRegister, $src$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister,
20915 $ktmp1$$KRegister, vec_enc);
20916 %}
20917 ins_pipe( pipe_slow );
20918 %}
20919
20920 // ---------------------------------------
20921 // For copySign use 0xE4 as writemask for vpternlog
20922 // Desired Truth Table: A -> xmm0 bit, B -> xmm1 bit, C -> xmm2 bit
20923 // C (xmm2) is set to 0x7FFFFFFF
20924 // Wherever xmm2 is 0, we want to pick from B (sign)
20925 // Wherever xmm2 is 1, we want to pick from A (src)
20926 //
20927 // A B C Result
20928 // 0 0 0 0
20929 // 0 0 1 0
20930 // 0 1 0 1
20931 // 0 1 1 0
20932 // 1 0 0 0
20933 // 1 0 1 1
20934 // 1 1 0 1
20935 // 1 1 1 1
20936 //
20937 // Result going from high bit to low bit is 0x11100100 = 0xe4
20938 // ---------------------------------------
20939
20940 instruct copySignF_reg(regF dst, regF src, regF tmp1, rRegI tmp2) %{
20941 match(Set dst (CopySignF dst src));
20942 effect(TEMP tmp1, TEMP tmp2);
20943 format %{ "CopySignF $dst, $src\t! using $tmp1 and $tmp2 as TEMP" %}
20944 ins_encode %{
20945 __ movl($tmp2$$Register, 0x7FFFFFFF);
20946 __ movdl($tmp1$$XMMRegister, $tmp2$$Register);
20947 __ vpternlogd($dst$$XMMRegister, 0xE4, $src$$XMMRegister, $tmp1$$XMMRegister, Assembler::AVX_128bit);
20948 %}
20949 ins_pipe( pipe_slow );
20950 %}
20951
20952 instruct copySignD_imm(regD dst, regD src, regD tmp1, rRegL tmp2, immD zero) %{
20953 match(Set dst (CopySignD dst (Binary src zero)));
20954 ins_cost(100);
20955 effect(TEMP tmp1, TEMP tmp2);
20956 format %{ "CopySignD $dst, $src\t! using $tmp1 and $tmp2 as TEMP" %}
20957 ins_encode %{
20958 __ mov64($tmp2$$Register, 0x7FFFFFFFFFFFFFFF);
20959 __ movq($tmp1$$XMMRegister, $tmp2$$Register);
20960 __ vpternlogq($dst$$XMMRegister, 0xE4, $src$$XMMRegister, $tmp1$$XMMRegister, Assembler::AVX_128bit);
20961 %}
20962 ins_pipe( pipe_slow );
20963 %}
20964
20965 //----------------------------- CompressBits/ExpandBits ------------------------
20966
20967 instruct compressBitsI_reg(rRegI dst, rRegI src, rRegI mask) %{
20968 predicate(n->bottom_type()->isa_int());
20969 match(Set dst (CompressBits src mask));
20970 format %{ "pextl $dst, $src, $mask\t! parallel bit extract" %}
20971 ins_encode %{
20972 __ pextl($dst$$Register, $src$$Register, $mask$$Register);
20973 %}
20974 ins_pipe( pipe_slow );
20975 %}
20976
20977 instruct expandBitsI_reg(rRegI dst, rRegI src, rRegI mask) %{
20978 predicate(n->bottom_type()->isa_int());
20979 match(Set dst (ExpandBits src mask));
20980 format %{ "pdepl $dst, $src, $mask\t! parallel bit deposit" %}
20981 ins_encode %{
20982 __ pdepl($dst$$Register, $src$$Register, $mask$$Register);
20983 %}
20984 ins_pipe( pipe_slow );
20985 %}
20986
20987 instruct compressBitsI_mem(rRegI dst, rRegI src, memory mask) %{
20988 predicate(n->bottom_type()->isa_int());
20989 match(Set dst (CompressBits src (LoadI mask)));
20990 format %{ "pextl $dst, $src, $mask\t! parallel bit extract" %}
20991 ins_encode %{
20992 __ pextl($dst$$Register, $src$$Register, $mask$$Address);
20993 %}
20994 ins_pipe( pipe_slow );
20995 %}
20996
20997 instruct expandBitsI_mem(rRegI dst, rRegI src, memory mask) %{
20998 predicate(n->bottom_type()->isa_int());
20999 match(Set dst (ExpandBits src (LoadI mask)));
21000 format %{ "pdepl $dst, $src, $mask\t! parallel bit deposit" %}
21001 ins_encode %{
21002 __ pdepl($dst$$Register, $src$$Register, $mask$$Address);
21003 %}
21004 ins_pipe( pipe_slow );
21005 %}
21006
21007 // --------------------------------- Sqrt --------------------------------------
21008
21009 instruct vsqrtF_reg(vec dst, vec src) %{
21010 match(Set dst (SqrtVF src));
21011 format %{ "vsqrtps $dst,$src\t! sqrt packedF" %}
21012 ins_encode %{
21013 assert(UseAVX > 0, "required");
21014 int vlen_enc = vector_length_encoding(this);
21015 __ vsqrtps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21016 %}
21017 ins_pipe( pipe_slow );
21018 %}
21019
21020 instruct vsqrtF_mem(vec dst, memory mem) %{
21021 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
21022 match(Set dst (SqrtVF (LoadVector mem)));
21023 format %{ "vsqrtps $dst,$mem\t! sqrt packedF" %}
21024 ins_encode %{
21025 assert(UseAVX > 0, "required");
21026 int vlen_enc = vector_length_encoding(this);
21027 __ vsqrtps($dst$$XMMRegister, $mem$$Address, vlen_enc);
21028 %}
21029 ins_pipe( pipe_slow );
21030 %}
21031
21032 // Floating point vector sqrt
21033 instruct vsqrtD_reg(vec dst, vec src) %{
21034 match(Set dst (SqrtVD src));
21035 format %{ "vsqrtpd $dst,$src\t! sqrt packedD" %}
21036 ins_encode %{
21037 assert(UseAVX > 0, "required");
21038 int vlen_enc = vector_length_encoding(this);
21039 __ vsqrtpd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21040 %}
21041 ins_pipe( pipe_slow );
21042 %}
21043
21044 instruct vsqrtD_mem(vec dst, memory mem) %{
21045 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
21046 match(Set dst (SqrtVD (LoadVector mem)));
21047 format %{ "vsqrtpd $dst,$mem\t! sqrt packedD" %}
21048 ins_encode %{
21049 assert(UseAVX > 0, "required");
21050 int vlen_enc = vector_length_encoding(this);
21051 __ vsqrtpd($dst$$XMMRegister, $mem$$Address, vlen_enc);
21052 %}
21053 ins_pipe( pipe_slow );
21054 %}
21055
21056 // ------------------------------ Shift ---------------------------------------
21057
21058 // Left and right shift count vectors are the same on x86
21059 // (only lowest bits of xmm reg are used for count).
21060 instruct vshiftcnt(vec dst, rRegI cnt) %{
21061 match(Set dst (LShiftCntV cnt));
21062 match(Set dst (RShiftCntV cnt));
21063 format %{ "movdl $dst,$cnt\t! load shift count" %}
21064 ins_encode %{
21065 __ movdl($dst$$XMMRegister, $cnt$$Register);
21066 %}
21067 ins_pipe( pipe_slow );
21068 %}
21069
21070 // Byte vector shift
21071 instruct vshiftB(vec dst, vec src, vec shift, vec tmp) %{
21072 predicate(Matcher::vector_length(n) <= 8 && !n->as_ShiftV()->is_var_shift());
21073 match(Set dst ( LShiftVB src shift));
21074 match(Set dst ( RShiftVB src shift));
21075 match(Set dst (URShiftVB src shift));
21076 effect(TEMP dst, USE src, USE shift, TEMP tmp);
21077 format %{"vector_byte_shift $dst,$src,$shift" %}
21078 ins_encode %{
21079 assert(UseSSE > 3, "required");
21080 int opcode = this->ideal_Opcode();
21081 bool sign = (opcode != Op_URShiftVB);
21082 __ vextendbw(sign, $tmp$$XMMRegister, $src$$XMMRegister);
21083 __ vshiftw(opcode, $tmp$$XMMRegister, $shift$$XMMRegister);
21084 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), noreg);
21085 __ pand($dst$$XMMRegister, $tmp$$XMMRegister);
21086 __ packuswb($dst$$XMMRegister, $dst$$XMMRegister);
21087 %}
21088 ins_pipe( pipe_slow );
21089 %}
21090
21091 instruct vshift16B(vec dst, vec src, vec shift, vec tmp1, vec tmp2) %{
21092 predicate(Matcher::vector_length(n) == 16 && !n->as_ShiftV()->is_var_shift() &&
21093 UseAVX <= 1);
21094 match(Set dst ( LShiftVB src shift));
21095 match(Set dst ( RShiftVB src shift));
21096 match(Set dst (URShiftVB src shift));
21097 effect(TEMP dst, USE src, USE shift, TEMP tmp1, TEMP tmp2);
21098 format %{"vector_byte_shift $dst,$src,$shift" %}
21099 ins_encode %{
21100 assert(UseSSE > 3, "required");
21101 int opcode = this->ideal_Opcode();
21102 bool sign = (opcode != Op_URShiftVB);
21103 __ vextendbw(sign, $tmp1$$XMMRegister, $src$$XMMRegister);
21104 __ vshiftw(opcode, $tmp1$$XMMRegister, $shift$$XMMRegister);
21105 __ pshufd($tmp2$$XMMRegister, $src$$XMMRegister, 0xE);
21106 __ vextendbw(sign, $tmp2$$XMMRegister, $tmp2$$XMMRegister);
21107 __ vshiftw(opcode, $tmp2$$XMMRegister, $shift$$XMMRegister);
21108 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), noreg);
21109 __ pand($tmp2$$XMMRegister, $dst$$XMMRegister);
21110 __ pand($dst$$XMMRegister, $tmp1$$XMMRegister);
21111 __ packuswb($dst$$XMMRegister, $tmp2$$XMMRegister);
21112 %}
21113 ins_pipe( pipe_slow );
21114 %}
21115
21116 instruct vshift16B_avx(vec dst, vec src, vec shift, vec tmp) %{
21117 predicate(Matcher::vector_length(n) == 16 && !n->as_ShiftV()->is_var_shift() &&
21118 UseAVX > 1);
21119 match(Set dst ( LShiftVB src shift));
21120 match(Set dst ( RShiftVB src shift));
21121 match(Set dst (URShiftVB src shift));
21122 effect(TEMP dst, TEMP tmp);
21123 format %{"vector_byte_shift $dst,$src,$shift" %}
21124 ins_encode %{
21125 int opcode = this->ideal_Opcode();
21126 bool sign = (opcode != Op_URShiftVB);
21127 int vlen_enc = Assembler::AVX_256bit;
21128 __ vextendbw(sign, $tmp$$XMMRegister, $src$$XMMRegister, vlen_enc);
21129 __ vshiftw(opcode, $tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21130 __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
21131 __ vextracti128_high($dst$$XMMRegister, $tmp$$XMMRegister);
21132 __ vpackuswb($dst$$XMMRegister, $tmp$$XMMRegister, $dst$$XMMRegister, 0);
21133 %}
21134 ins_pipe( pipe_slow );
21135 %}
21136
21137 instruct vshift32B_avx(vec dst, vec src, vec shift, vec tmp) %{
21138 predicate(Matcher::vector_length(n) == 32 && !n->as_ShiftV()->is_var_shift());
21139 match(Set dst ( LShiftVB src shift));
21140 match(Set dst ( RShiftVB src shift));
21141 match(Set dst (URShiftVB src shift));
21142 effect(TEMP dst, TEMP tmp);
21143 format %{"vector_byte_shift $dst,$src,$shift" %}
21144 ins_encode %{
21145 assert(UseAVX > 1, "required");
21146 int opcode = this->ideal_Opcode();
21147 bool sign = (opcode != Op_URShiftVB);
21148 int vlen_enc = Assembler::AVX_256bit;
21149 __ vextracti128_high($tmp$$XMMRegister, $src$$XMMRegister);
21150 __ vextendbw(sign, $tmp$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
21151 __ vextendbw(sign, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21152 __ vshiftw(opcode, $tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21153 __ vshiftw(opcode, $dst$$XMMRegister, $dst$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21154 __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
21155 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
21156 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
21157 __ vpermq($dst$$XMMRegister, $dst$$XMMRegister, 0xD8, vlen_enc);
21158 %}
21159 ins_pipe( pipe_slow );
21160 %}
21161
21162 instruct vshift64B_avx(vec dst, vec src, vec shift, vec tmp1, vec tmp2) %{
21163 predicate(Matcher::vector_length(n) == 64 && !n->as_ShiftV()->is_var_shift());
21164 match(Set dst ( LShiftVB src shift));
21165 match(Set dst (RShiftVB src shift));
21166 match(Set dst (URShiftVB src shift));
21167 effect(TEMP dst, TEMP tmp1, TEMP tmp2);
21168 format %{"vector_byte_shift $dst,$src,$shift" %}
21169 ins_encode %{
21170 assert(UseAVX > 2, "required");
21171 int opcode = this->ideal_Opcode();
21172 bool sign = (opcode != Op_URShiftVB);
21173 int vlen_enc = Assembler::AVX_512bit;
21174 __ vextracti64x4($tmp1$$XMMRegister, $src$$XMMRegister, 1);
21175 __ vextendbw(sign, $tmp1$$XMMRegister, $tmp1$$XMMRegister, vlen_enc);
21176 __ vextendbw(sign, $tmp2$$XMMRegister, $src$$XMMRegister, vlen_enc);
21177 __ vshiftw(opcode, $tmp1$$XMMRegister, $tmp1$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21178 __ vshiftw(opcode, $tmp2$$XMMRegister, $tmp2$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21179 __ vmovdqu($dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), noreg);
21180 __ vpbroadcastd($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21181 __ vpand($tmp1$$XMMRegister, $tmp1$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21182 __ vpand($tmp2$$XMMRegister, $tmp2$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21183 __ vpackuswb($dst$$XMMRegister, $tmp1$$XMMRegister, $tmp2$$XMMRegister, vlen_enc);
21184 __ evmovdquq($tmp2$$XMMRegister, ExternalAddress(vector_byte_perm_mask()), vlen_enc, noreg);
21185 __ vpermq($dst$$XMMRegister, $tmp2$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21186 %}
21187 ins_pipe( pipe_slow );
21188 %}
21189
21190 // Shorts vector logical right shift produces incorrect Java result
21191 // for negative data because java code convert short value into int with
21192 // sign extension before a shift. But char vectors are fine since chars are
21193 // unsigned values.
21194 // Shorts/Chars vector left shift
21195 instruct vshiftS(vec dst, vec src, vec shift) %{
21196 predicate(!n->as_ShiftV()->is_var_shift());
21197 match(Set dst ( LShiftVS src shift));
21198 match(Set dst ( RShiftVS src shift));
21199 match(Set dst (URShiftVS src shift));
21200 effect(TEMP dst, USE src, USE shift);
21201 format %{ "vshiftw $dst,$src,$shift\t! shift packedS" %}
21202 ins_encode %{
21203 int opcode = this->ideal_Opcode();
21204 if (UseAVX > 0) {
21205 int vlen_enc = vector_length_encoding(this);
21206 __ vshiftw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21207 } else {
21208 int vlen = Matcher::vector_length(this);
21209 if (vlen == 2) {
21210 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
21211 __ vshiftw(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
21212 } else if (vlen == 4) {
21213 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
21214 __ vshiftw(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
21215 } else {
21216 assert (vlen == 8, "sanity");
21217 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
21218 __ vshiftw(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
21219 }
21220 }
21221 %}
21222 ins_pipe( pipe_slow );
21223 %}
21224
21225 // Integers vector left shift
21226 instruct vshiftI(vec dst, vec src, vec shift) %{
21227 predicate(!n->as_ShiftV()->is_var_shift());
21228 match(Set dst ( LShiftVI src shift));
21229 match(Set dst ( RShiftVI src shift));
21230 match(Set dst (URShiftVI src shift));
21231 effect(TEMP dst, USE src, USE shift);
21232 format %{ "vshiftd $dst,$src,$shift\t! shift packedI" %}
21233 ins_encode %{
21234 int opcode = this->ideal_Opcode();
21235 if (UseAVX > 0) {
21236 int vlen_enc = vector_length_encoding(this);
21237 __ vshiftd(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21238 } else {
21239 int vlen = Matcher::vector_length(this);
21240 if (vlen == 2) {
21241 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
21242 __ vshiftd(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
21243 } else {
21244 assert(vlen == 4, "sanity");
21245 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
21246 __ vshiftd(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
21247 }
21248 }
21249 %}
21250 ins_pipe( pipe_slow );
21251 %}
21252
21253 // Integers vector left constant shift
21254 instruct vshiftI_imm(vec dst, vec src, immI8 shift) %{
21255 match(Set dst (LShiftVI src (LShiftCntV shift)));
21256 match(Set dst (RShiftVI src (RShiftCntV shift)));
21257 match(Set dst (URShiftVI src (RShiftCntV shift)));
21258 format %{ "vshiftd_imm $dst,$src,$shift\t! shift packedI" %}
21259 ins_encode %{
21260 int opcode = this->ideal_Opcode();
21261 if (UseAVX > 0) {
21262 int vector_len = vector_length_encoding(this);
21263 __ vshiftd_imm(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$constant, vector_len);
21264 } else {
21265 int vlen = Matcher::vector_length(this);
21266 if (vlen == 2) {
21267 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
21268 __ vshiftd_imm(opcode, $dst$$XMMRegister, $shift$$constant);
21269 } else {
21270 assert(vlen == 4, "sanity");
21271 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
21272 __ vshiftd_imm(opcode, $dst$$XMMRegister, $shift$$constant);
21273 }
21274 }
21275 %}
21276 ins_pipe( pipe_slow );
21277 %}
21278
21279 // Longs vector shift
21280 instruct vshiftL(vec dst, vec src, vec shift) %{
21281 predicate(!n->as_ShiftV()->is_var_shift());
21282 match(Set dst ( LShiftVL src shift));
21283 match(Set dst (URShiftVL src shift));
21284 effect(TEMP dst, USE src, USE shift);
21285 format %{ "vshiftq $dst,$src,$shift\t! shift packedL" %}
21286 ins_encode %{
21287 int opcode = this->ideal_Opcode();
21288 if (UseAVX > 0) {
21289 int vlen_enc = vector_length_encoding(this);
21290 __ vshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21291 } else {
21292 assert(Matcher::vector_length(this) == 2, "");
21293 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
21294 __ vshiftq(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
21295 }
21296 %}
21297 ins_pipe( pipe_slow );
21298 %}
21299
21300 // Longs vector constant shift
21301 instruct vshiftL_imm(vec dst, vec src, immI8 shift) %{
21302 match(Set dst (LShiftVL src (LShiftCntV shift)));
21303 match(Set dst (URShiftVL src (RShiftCntV shift)));
21304 format %{ "vshiftq_imm $dst,$src,$shift\t! shift packedL" %}
21305 ins_encode %{
21306 int opcode = this->ideal_Opcode();
21307 if (UseAVX > 0) {
21308 int vector_len = vector_length_encoding(this);
21309 __ vshiftq_imm(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$constant, vector_len);
21310 } else {
21311 assert(Matcher::vector_length(this) == 2, "");
21312 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
21313 __ vshiftq_imm(opcode, $dst$$XMMRegister, $shift$$constant);
21314 }
21315 %}
21316 ins_pipe( pipe_slow );
21317 %}
21318
21319 // -------------------ArithmeticRightShift -----------------------------------
21320 // Long vector arithmetic right shift
21321 instruct vshiftL_arith_reg(vec dst, vec src, vec shift, vec tmp) %{
21322 predicate(!n->as_ShiftV()->is_var_shift() && UseAVX <= 2);
21323 match(Set dst (RShiftVL src shift));
21324 effect(TEMP dst, TEMP tmp);
21325 format %{ "vshiftq $dst,$src,$shift" %}
21326 ins_encode %{
21327 uint vlen = Matcher::vector_length(this);
21328 if (vlen == 2) {
21329 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
21330 __ psrlq($dst$$XMMRegister, $shift$$XMMRegister);
21331 __ movdqu($tmp$$XMMRegister, ExternalAddress(vector_long_sign_mask()), noreg);
21332 __ psrlq($tmp$$XMMRegister, $shift$$XMMRegister);
21333 __ pxor($dst$$XMMRegister, $tmp$$XMMRegister);
21334 __ psubq($dst$$XMMRegister, $tmp$$XMMRegister);
21335 } else {
21336 assert(vlen == 4, "sanity");
21337 assert(UseAVX > 1, "required");
21338 int vlen_enc = Assembler::AVX_256bit;
21339 __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21340 __ vmovdqu($tmp$$XMMRegister, ExternalAddress(vector_long_sign_mask()), noreg);
21341 __ vpsrlq($tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21342 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
21343 __ vpsubq($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
21344 }
21345 %}
21346 ins_pipe( pipe_slow );
21347 %}
21348
21349 instruct vshiftL_arith_reg_evex(vec dst, vec src, vec shift) %{
21350 predicate(!n->as_ShiftV()->is_var_shift() && UseAVX > 2);
21351 match(Set dst (RShiftVL src shift));
21352 format %{ "vshiftq $dst,$src,$shift" %}
21353 ins_encode %{
21354 int vlen_enc = vector_length_encoding(this);
21355 __ evpsraq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21356 %}
21357 ins_pipe( pipe_slow );
21358 %}
21359
21360 // ------------------- Variable Shift -----------------------------
21361 // Byte variable shift
21362 instruct vshift8B_var_nobw(vec dst, vec src, vec shift, vec vtmp) %{
21363 predicate(Matcher::vector_length(n) <= 8 &&
21364 n->as_ShiftV()->is_var_shift() &&
21365 !VM_Version::supports_avx512bw());
21366 match(Set dst ( LShiftVB src shift));
21367 match(Set dst ( RShiftVB src shift));
21368 match(Set dst (URShiftVB src shift));
21369 effect(TEMP dst, TEMP vtmp);
21370 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp as TEMP" %}
21371 ins_encode %{
21372 assert(UseAVX >= 2, "required");
21373
21374 int opcode = this->ideal_Opcode();
21375 int vlen_enc = Assembler::AVX_128bit;
21376 __ varshiftbw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp$$XMMRegister);
21377 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, 0);
21378 %}
21379 ins_pipe( pipe_slow );
21380 %}
21381
21382 instruct vshift16B_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2) %{
21383 predicate(Matcher::vector_length(n) == 16 &&
21384 n->as_ShiftV()->is_var_shift() &&
21385 !VM_Version::supports_avx512bw());
21386 match(Set dst ( LShiftVB src shift));
21387 match(Set dst ( RShiftVB src shift));
21388 match(Set dst (URShiftVB src shift));
21389 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
21390 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp1, $vtmp2 as TEMP" %}
21391 ins_encode %{
21392 assert(UseAVX >= 2, "required");
21393
21394 int opcode = this->ideal_Opcode();
21395 int vlen_enc = Assembler::AVX_128bit;
21396 // Shift lower half and get word result in dst
21397 __ varshiftbw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp1$$XMMRegister);
21398
21399 // Shift upper half and get word result in vtmp1
21400 __ vpshufd($vtmp1$$XMMRegister, $src$$XMMRegister, 0xE, 0);
21401 __ vpshufd($vtmp2$$XMMRegister, $shift$$XMMRegister, 0xE, 0);
21402 __ varshiftbw(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21403
21404 // Merge and down convert the two word results to byte in dst
21405 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0);
21406 %}
21407 ins_pipe( pipe_slow );
21408 %}
21409
21410 instruct vshift32B_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2, vec vtmp3, vec vtmp4) %{
21411 predicate(Matcher::vector_length(n) == 32 &&
21412 n->as_ShiftV()->is_var_shift() &&
21413 !VM_Version::supports_avx512bw());
21414 match(Set dst ( LShiftVB src shift));
21415 match(Set dst ( RShiftVB src shift));
21416 match(Set dst (URShiftVB src shift));
21417 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4);
21418 format %{ "vector_varshift_byte $dst, $src, $shift\n\t using $vtmp1, $vtmp2, $vtmp3, $vtmp4 as TEMP" %}
21419 ins_encode %{
21420 assert(UseAVX >= 2, "required");
21421
21422 int opcode = this->ideal_Opcode();
21423 int vlen_enc = Assembler::AVX_128bit;
21424 // Process lower 128 bits and get result in dst
21425 __ varshiftbw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp1$$XMMRegister);
21426 __ vpshufd($vtmp1$$XMMRegister, $src$$XMMRegister, 0xE, 0);
21427 __ vpshufd($vtmp2$$XMMRegister, $shift$$XMMRegister, 0xE, 0);
21428 __ varshiftbw(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21429 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0);
21430
21431 // Process higher 128 bits and get result in vtmp3
21432 __ vextracti128_high($vtmp1$$XMMRegister, $src$$XMMRegister);
21433 __ vextracti128_high($vtmp2$$XMMRegister, $shift$$XMMRegister);
21434 __ varshiftbw(opcode, $vtmp3$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp4$$XMMRegister);
21435 __ vpshufd($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, 0xE, 0);
21436 __ vpshufd($vtmp2$$XMMRegister, $vtmp2$$XMMRegister, 0xE, 0);
21437 __ varshiftbw(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21438 __ vpackuswb($vtmp1$$XMMRegister, $vtmp3$$XMMRegister, $vtmp1$$XMMRegister, 0);
21439
21440 // Merge the two results in dst
21441 __ vinserti128($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0x1);
21442 %}
21443 ins_pipe( pipe_slow );
21444 %}
21445
21446 instruct vshiftB_var_evex_bw(vec dst, vec src, vec shift, vec vtmp) %{
21447 predicate(Matcher::vector_length(n) <= 32 &&
21448 n->as_ShiftV()->is_var_shift() &&
21449 VM_Version::supports_avx512bw());
21450 match(Set dst ( LShiftVB src shift));
21451 match(Set dst ( RShiftVB src shift));
21452 match(Set dst (URShiftVB src shift));
21453 effect(TEMP dst, TEMP vtmp);
21454 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp as TEMP" %}
21455 ins_encode %{
21456 assert(UseAVX > 2, "required");
21457
21458 int opcode = this->ideal_Opcode();
21459 int vlen_enc = vector_length_encoding(this);
21460 __ evarshiftb(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp$$XMMRegister);
21461 %}
21462 ins_pipe( pipe_slow );
21463 %}
21464
21465 instruct vshift64B_var_evex_bw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2) %{
21466 predicate(Matcher::vector_length(n) == 64 &&
21467 n->as_ShiftV()->is_var_shift() &&
21468 VM_Version::supports_avx512bw());
21469 match(Set dst ( LShiftVB src shift));
21470 match(Set dst ( RShiftVB src shift));
21471 match(Set dst (URShiftVB src shift));
21472 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
21473 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp1, $vtmp2 as TEMP" %}
21474 ins_encode %{
21475 assert(UseAVX > 2, "required");
21476
21477 int opcode = this->ideal_Opcode();
21478 int vlen_enc = Assembler::AVX_256bit;
21479 __ evarshiftb(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp1$$XMMRegister);
21480 __ vextracti64x4_high($vtmp1$$XMMRegister, $src$$XMMRegister);
21481 __ vextracti64x4_high($vtmp2$$XMMRegister, $shift$$XMMRegister);
21482 __ evarshiftb(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21483 __ vinserti64x4($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0x1);
21484 %}
21485 ins_pipe( pipe_slow );
21486 %}
21487
21488 // Short variable shift
21489 instruct vshift8S_var_nobw(vec dst, vec src, vec shift, vec vtmp) %{
21490 predicate(Matcher::vector_length(n) <= 8 &&
21491 n->as_ShiftV()->is_var_shift() &&
21492 !VM_Version::supports_avx512bw());
21493 match(Set dst ( LShiftVS src shift));
21494 match(Set dst ( RShiftVS src shift));
21495 match(Set dst (URShiftVS src shift));
21496 effect(TEMP dst, TEMP vtmp);
21497 format %{ "vector_var_shift_left_short $dst, $src, $shift\n\t" %}
21498 ins_encode %{
21499 assert(UseAVX >= 2, "required");
21500
21501 int opcode = this->ideal_Opcode();
21502 bool sign = (opcode != Op_URShiftVS);
21503 int vlen_enc = Assembler::AVX_256bit;
21504 __ vextendwd(sign, $dst$$XMMRegister, $src$$XMMRegister, 1);
21505 __ vpmovzxwd($vtmp$$XMMRegister, $shift$$XMMRegister, 1);
21506 __ varshiftd(opcode, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
21507 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21508 __ vextracti128_high($vtmp$$XMMRegister, $dst$$XMMRegister);
21509 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0);
21510 %}
21511 ins_pipe( pipe_slow );
21512 %}
21513
21514 instruct vshift16S_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2) %{
21515 predicate(Matcher::vector_length(n) == 16 &&
21516 n->as_ShiftV()->is_var_shift() &&
21517 !VM_Version::supports_avx512bw());
21518 match(Set dst ( LShiftVS src shift));
21519 match(Set dst ( RShiftVS src shift));
21520 match(Set dst (URShiftVS src shift));
21521 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
21522 format %{ "vector_var_shift_left_short $dst, $src, $shift\n\t" %}
21523 ins_encode %{
21524 assert(UseAVX >= 2, "required");
21525
21526 int opcode = this->ideal_Opcode();
21527 bool sign = (opcode != Op_URShiftVS);
21528 int vlen_enc = Assembler::AVX_256bit;
21529 // Shift lower half, with result in vtmp2 using vtmp1 as TEMP
21530 __ vextendwd(sign, $vtmp2$$XMMRegister, $src$$XMMRegister, vlen_enc);
21531 __ vpmovzxwd($vtmp1$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21532 __ varshiftd(opcode, $vtmp2$$XMMRegister, $vtmp2$$XMMRegister, $vtmp1$$XMMRegister, vlen_enc);
21533 __ vpand($vtmp2$$XMMRegister, $vtmp2$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21534
21535 // Shift upper half, with result in dst using vtmp1 as TEMP
21536 __ vextracti128_high($dst$$XMMRegister, $src$$XMMRegister);
21537 __ vextracti128_high($vtmp1$$XMMRegister, $shift$$XMMRegister);
21538 __ vextendwd(sign, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21539 __ vpmovzxwd($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, vlen_enc);
21540 __ varshiftd(opcode, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, vlen_enc);
21541 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21542
21543 // Merge lower and upper half result into dst
21544 __ vpackusdw($dst$$XMMRegister, $vtmp2$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21545 __ vpermq($dst$$XMMRegister, $dst$$XMMRegister, 0xD8, vlen_enc);
21546 %}
21547 ins_pipe( pipe_slow );
21548 %}
21549
21550 instruct vshift16S_var_evex_bw(vec dst, vec src, vec shift) %{
21551 predicate(n->as_ShiftV()->is_var_shift() &&
21552 VM_Version::supports_avx512bw());
21553 match(Set dst ( LShiftVS src shift));
21554 match(Set dst ( RShiftVS src shift));
21555 match(Set dst (URShiftVS src shift));
21556 format %{ "vector_varshift_short $dst,$src,$shift\t!" %}
21557 ins_encode %{
21558 assert(UseAVX > 2, "required");
21559
21560 int opcode = this->ideal_Opcode();
21561 int vlen_enc = vector_length_encoding(this);
21562 if (!VM_Version::supports_avx512vl()) {
21563 vlen_enc = Assembler::AVX_512bit;
21564 }
21565 __ varshiftw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21566 %}
21567 ins_pipe( pipe_slow );
21568 %}
21569
21570 //Integer variable shift
21571 instruct vshiftI_var(vec dst, vec src, vec shift) %{
21572 predicate(n->as_ShiftV()->is_var_shift());
21573 match(Set dst ( LShiftVI src shift));
21574 match(Set dst ( RShiftVI src shift));
21575 match(Set dst (URShiftVI src shift));
21576 format %{ "vector_varshift_int $dst,$src,$shift\t!" %}
21577 ins_encode %{
21578 assert(UseAVX >= 2, "required");
21579
21580 int opcode = this->ideal_Opcode();
21581 int vlen_enc = vector_length_encoding(this);
21582 __ varshiftd(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21583 %}
21584 ins_pipe( pipe_slow );
21585 %}
21586
21587 //Long variable shift
21588 instruct vshiftL_var(vec dst, vec src, vec shift) %{
21589 predicate(n->as_ShiftV()->is_var_shift());
21590 match(Set dst ( LShiftVL src shift));
21591 match(Set dst (URShiftVL src shift));
21592 format %{ "vector_varshift_long $dst,$src,$shift\t!" %}
21593 ins_encode %{
21594 assert(UseAVX >= 2, "required");
21595
21596 int opcode = this->ideal_Opcode();
21597 int vlen_enc = vector_length_encoding(this);
21598 __ varshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21599 %}
21600 ins_pipe( pipe_slow );
21601 %}
21602
21603 //Long variable right shift arithmetic
21604 instruct vshiftL_arith_var(vec dst, vec src, vec shift, vec vtmp) %{
21605 predicate(Matcher::vector_length(n) <= 4 &&
21606 n->as_ShiftV()->is_var_shift() &&
21607 UseAVX == 2);
21608 match(Set dst (RShiftVL src shift));
21609 effect(TEMP dst, TEMP vtmp);
21610 format %{ "vector_varshift_long $dst,$src,$shift\n\t! using $vtmp as TEMP" %}
21611 ins_encode %{
21612 int opcode = this->ideal_Opcode();
21613 int vlen_enc = vector_length_encoding(this);
21614 __ varshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc,
21615 $vtmp$$XMMRegister);
21616 %}
21617 ins_pipe( pipe_slow );
21618 %}
21619
21620 instruct vshiftL_arith_var_evex(vec dst, vec src, vec shift) %{
21621 predicate(n->as_ShiftV()->is_var_shift() &&
21622 UseAVX > 2);
21623 match(Set dst (RShiftVL src shift));
21624 format %{ "vector_varfshift_long $dst,$src,$shift\t!" %}
21625 ins_encode %{
21626 int opcode = this->ideal_Opcode();
21627 int vlen_enc = vector_length_encoding(this);
21628 __ varshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21629 %}
21630 ins_pipe( pipe_slow );
21631 %}
21632
21633 // --------------------------------- AND --------------------------------------
21634
21635 instruct vand(vec dst, vec src) %{
21636 predicate(UseAVX == 0);
21637 match(Set dst (AndV dst src));
21638 format %{ "pand $dst,$src\t! and vectors" %}
21639 ins_encode %{
21640 __ pand($dst$$XMMRegister, $src$$XMMRegister);
21641 %}
21642 ins_pipe( pipe_slow );
21643 %}
21644
21645 instruct vand_reg(vec dst, vec src1, vec src2) %{
21646 predicate(UseAVX > 0);
21647 match(Set dst (AndV src1 src2));
21648 format %{ "vpand $dst,$src1,$src2\t! and vectors" %}
21649 ins_encode %{
21650 int vlen_enc = vector_length_encoding(this);
21651 __ vpand($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
21652 %}
21653 ins_pipe( pipe_slow );
21654 %}
21655
21656 instruct vand_mem(vec dst, vec src, memory mem) %{
21657 predicate((UseAVX > 0) &&
21658 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
21659 match(Set dst (AndV src (LoadVector mem)));
21660 format %{ "vpand $dst,$src,$mem\t! and vectors" %}
21661 ins_encode %{
21662 int vlen_enc = vector_length_encoding(this);
21663 __ vpand($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
21664 %}
21665 ins_pipe( pipe_slow );
21666 %}
21667
21668 // --------------------------------- OR ---------------------------------------
21669
21670 instruct vor(vec dst, vec src) %{
21671 predicate(UseAVX == 0);
21672 match(Set dst (OrV dst src));
21673 format %{ "por $dst,$src\t! or vectors" %}
21674 ins_encode %{
21675 __ por($dst$$XMMRegister, $src$$XMMRegister);
21676 %}
21677 ins_pipe( pipe_slow );
21678 %}
21679
21680 instruct vor_reg(vec dst, vec src1, vec src2) %{
21681 predicate(UseAVX > 0);
21682 match(Set dst (OrV src1 src2));
21683 format %{ "vpor $dst,$src1,$src2\t! or vectors" %}
21684 ins_encode %{
21685 int vlen_enc = vector_length_encoding(this);
21686 __ vpor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
21687 %}
21688 ins_pipe( pipe_slow );
21689 %}
21690
21691 instruct vor_mem(vec dst, vec src, memory mem) %{
21692 predicate((UseAVX > 0) &&
21693 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
21694 match(Set dst (OrV src (LoadVector mem)));
21695 format %{ "vpor $dst,$src,$mem\t! or vectors" %}
21696 ins_encode %{
21697 int vlen_enc = vector_length_encoding(this);
21698 __ vpor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
21699 %}
21700 ins_pipe( pipe_slow );
21701 %}
21702
21703 // --------------------------------- XOR --------------------------------------
21704
21705 instruct vxor(vec dst, vec src) %{
21706 predicate(UseAVX == 0);
21707 match(Set dst (XorV dst src));
21708 format %{ "pxor $dst,$src\t! xor vectors" %}
21709 ins_encode %{
21710 __ pxor($dst$$XMMRegister, $src$$XMMRegister);
21711 %}
21712 ins_pipe( pipe_slow );
21713 %}
21714
21715 instruct vxor_reg(vec dst, vec src1, vec src2) %{
21716 predicate(UseAVX > 0);
21717 match(Set dst (XorV src1 src2));
21718 format %{ "vpxor $dst,$src1,$src2\t! xor vectors" %}
21719 ins_encode %{
21720 int vlen_enc = vector_length_encoding(this);
21721 __ vpxor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
21722 %}
21723 ins_pipe( pipe_slow );
21724 %}
21725
21726 instruct vxor_mem(vec dst, vec src, memory mem) %{
21727 predicate((UseAVX > 0) &&
21728 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
21729 match(Set dst (XorV src (LoadVector mem)));
21730 format %{ "vpxor $dst,$src,$mem\t! xor vectors" %}
21731 ins_encode %{
21732 int vlen_enc = vector_length_encoding(this);
21733 __ vpxor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
21734 %}
21735 ins_pipe( pipe_slow );
21736 %}
21737
21738 // --------------------------------- VectorCast --------------------------------------
21739
21740 instruct vcastBtoX(vec dst, vec src) %{
21741 predicate(VM_Version::supports_avx512vl() || Matcher::vector_element_basic_type(n) != T_DOUBLE);
21742 match(Set dst (VectorCastB2X src));
21743 format %{ "vector_cast_b2x $dst,$src\t!" %}
21744 ins_encode %{
21745 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21746 int vlen_enc = vector_length_encoding(this);
21747 __ vconvert_b2x(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21748 %}
21749 ins_pipe( pipe_slow );
21750 %}
21751
21752 instruct vcastBtoD(legVec dst, legVec src) %{
21753 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_element_basic_type(n) == T_DOUBLE);
21754 match(Set dst (VectorCastB2X src));
21755 format %{ "vector_cast_b2x $dst,$src\t!" %}
21756 ins_encode %{
21757 int vlen_enc = vector_length_encoding(this);
21758 __ vconvert_b2x(T_DOUBLE, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21759 %}
21760 ins_pipe( pipe_slow );
21761 %}
21762
21763 instruct castStoX(vec dst, vec src) %{
21764 predicate((UseAVX <= 2 || !VM_Version::supports_avx512vlbw()) &&
21765 Matcher::vector_length(n->in(1)) <= 8 && // src
21766 Matcher::vector_element_basic_type(n) == T_BYTE);
21767 match(Set dst (VectorCastS2X src));
21768 format %{ "vector_cast_s2x $dst,$src" %}
21769 ins_encode %{
21770 assert(UseAVX > 0, "required");
21771
21772 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), 0, noreg);
21773 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, 0);
21774 %}
21775 ins_pipe( pipe_slow );
21776 %}
21777
21778 instruct vcastStoX(vec dst, vec src, vec vtmp) %{
21779 predicate((UseAVX <= 2 || !VM_Version::supports_avx512vlbw()) &&
21780 Matcher::vector_length(n->in(1)) == 16 && // src
21781 Matcher::vector_element_basic_type(n) == T_BYTE);
21782 effect(TEMP dst, TEMP vtmp);
21783 match(Set dst (VectorCastS2X src));
21784 format %{ "vector_cast_s2x $dst,$src\t! using $vtmp as TEMP" %}
21785 ins_encode %{
21786 assert(UseAVX > 0, "required");
21787
21788 int vlen_enc = vector_length_encoding(Matcher::vector_length_in_bytes(this, $src));
21789 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
21790 __ vextracti128($vtmp$$XMMRegister, $dst$$XMMRegister, 0x1);
21791 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0);
21792 %}
21793 ins_pipe( pipe_slow );
21794 %}
21795
21796 instruct vcastStoX_evex(vec dst, vec src) %{
21797 predicate((UseAVX > 2 && VM_Version::supports_avx512vlbw()) ||
21798 (Matcher::vector_length_in_bytes(n) >= Matcher::vector_length_in_bytes(n->in(1)))); // dst >= src
21799 match(Set dst (VectorCastS2X src));
21800 format %{ "vector_cast_s2x $dst,$src\t!" %}
21801 ins_encode %{
21802 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21803 int src_vlen_enc = vector_length_encoding(this, $src);
21804 int vlen_enc = vector_length_encoding(this);
21805 switch (to_elem_bt) {
21806 case T_BYTE:
21807 if (!VM_Version::supports_avx512vl()) {
21808 vlen_enc = Assembler::AVX_512bit;
21809 }
21810 __ evpmovwb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
21811 break;
21812 case T_INT:
21813 __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21814 break;
21815 case T_FLOAT:
21816 __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21817 __ vcvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21818 break;
21819 case T_LONG:
21820 __ vpmovsxwq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21821 break;
21822 case T_DOUBLE: {
21823 int mid_vlen_enc = (vlen_enc == Assembler::AVX_512bit) ? Assembler::AVX_256bit : Assembler::AVX_128bit;
21824 __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, mid_vlen_enc);
21825 __ vcvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21826 break;
21827 }
21828 default:
21829 ShouldNotReachHere();
21830 }
21831 %}
21832 ins_pipe( pipe_slow );
21833 %}
21834
21835 instruct castItoX(vec dst, vec src) %{
21836 predicate(UseAVX <= 2 &&
21837 (Matcher::vector_length_in_bytes(n->in(1)) <= 16) &&
21838 (Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)))); // dst < src
21839 match(Set dst (VectorCastI2X src));
21840 format %{ "vector_cast_i2x $dst,$src" %}
21841 ins_encode %{
21842 assert(UseAVX > 0, "required");
21843
21844 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21845 int vlen_enc = vector_length_encoding(this, $src);
21846
21847 if (to_elem_bt == T_BYTE) {
21848 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_byte_mask()), vlen_enc, noreg);
21849 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21850 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21851 } else {
21852 assert(to_elem_bt == T_SHORT, "%s", type2name(to_elem_bt));
21853 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21854 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21855 }
21856 %}
21857 ins_pipe( pipe_slow );
21858 %}
21859
21860 instruct vcastItoX(vec dst, vec src, vec vtmp) %{
21861 predicate(UseAVX <= 2 &&
21862 (Matcher::vector_length_in_bytes(n->in(1)) == 32) &&
21863 (Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)))); // dst < src
21864 match(Set dst (VectorCastI2X src));
21865 format %{ "vector_cast_i2x $dst,$src\t! using $vtmp as TEMP" %}
21866 effect(TEMP dst, TEMP vtmp);
21867 ins_encode %{
21868 assert(UseAVX > 0, "required");
21869
21870 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21871 int vlen_enc = vector_length_encoding(this, $src);
21872
21873 if (to_elem_bt == T_BYTE) {
21874 __ vpand($vtmp$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_byte_mask()), vlen_enc, noreg);
21875 __ vextracti128($dst$$XMMRegister, $vtmp$$XMMRegister, 0x1);
21876 __ vpackusdw($dst$$XMMRegister, $vtmp$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21877 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21878 } else {
21879 assert(to_elem_bt == T_SHORT, "%s", type2name(to_elem_bt));
21880 __ vpand($vtmp$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21881 __ vextracti128($dst$$XMMRegister, $vtmp$$XMMRegister, 0x1);
21882 __ vpackusdw($dst$$XMMRegister, $vtmp$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21883 }
21884 %}
21885 ins_pipe( pipe_slow );
21886 %}
21887
21888 instruct vcastItoX_evex(vec dst, vec src) %{
21889 predicate(UseAVX > 2 ||
21890 (Matcher::vector_length_in_bytes(n) >= Matcher::vector_length_in_bytes(n->in(1)))); // dst >= src
21891 match(Set dst (VectorCastI2X src));
21892 format %{ "vector_cast_i2x $dst,$src\t!" %}
21893 ins_encode %{
21894 assert(UseAVX > 0, "required");
21895
21896 BasicType dst_elem_bt = Matcher::vector_element_basic_type(this);
21897 int src_vlen_enc = vector_length_encoding(this, $src);
21898 int dst_vlen_enc = vector_length_encoding(this);
21899 switch (dst_elem_bt) {
21900 case T_BYTE:
21901 if (!VM_Version::supports_avx512vl()) {
21902 src_vlen_enc = Assembler::AVX_512bit;
21903 }
21904 __ evpmovdb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
21905 break;
21906 case T_SHORT:
21907 if (!VM_Version::supports_avx512vl()) {
21908 src_vlen_enc = Assembler::AVX_512bit;
21909 }
21910 __ evpmovdw($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
21911 break;
21912 case T_FLOAT:
21913 __ vcvtdq2ps($dst$$XMMRegister, $src$$XMMRegister, dst_vlen_enc);
21914 break;
21915 case T_LONG:
21916 __ vpmovsxdq($dst$$XMMRegister, $src$$XMMRegister, dst_vlen_enc);
21917 break;
21918 case T_DOUBLE:
21919 __ vcvtdq2pd($dst$$XMMRegister, $src$$XMMRegister, dst_vlen_enc);
21920 break;
21921 default:
21922 ShouldNotReachHere();
21923 }
21924 %}
21925 ins_pipe( pipe_slow );
21926 %}
21927
21928 instruct vcastLtoBS(vec dst, vec src) %{
21929 predicate((Matcher::vector_element_basic_type(n) == T_BYTE || Matcher::vector_element_basic_type(n) == T_SHORT) &&
21930 UseAVX <= 2);
21931 match(Set dst (VectorCastL2X src));
21932 format %{ "vector_cast_l2x $dst,$src" %}
21933 ins_encode %{
21934 assert(UseAVX > 0, "required");
21935
21936 int vlen = Matcher::vector_length_in_bytes(this, $src);
21937 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21938 AddressLiteral mask_addr = (to_elem_bt == T_BYTE) ? ExternalAddress(vector_int_to_byte_mask())
21939 : ExternalAddress(vector_int_to_short_mask());
21940 if (vlen <= 16) {
21941 __ vpshufd($dst$$XMMRegister, $src$$XMMRegister, 8, Assembler::AVX_128bit);
21942 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, mask_addr, Assembler::AVX_128bit, noreg);
21943 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21944 } else {
21945 assert(vlen <= 32, "required");
21946 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 8, Assembler::AVX_256bit);
21947 __ vpermpd($dst$$XMMRegister, $dst$$XMMRegister, 8, Assembler::AVX_256bit);
21948 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, mask_addr, Assembler::AVX_128bit, noreg);
21949 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21950 }
21951 if (to_elem_bt == T_BYTE) {
21952 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21953 }
21954 %}
21955 ins_pipe( pipe_slow );
21956 %}
21957
21958 instruct vcastLtoX_evex(vec dst, vec src) %{
21959 predicate(UseAVX > 2 ||
21960 (Matcher::vector_element_basic_type(n) == T_INT ||
21961 Matcher::vector_element_basic_type(n) == T_FLOAT ||
21962 Matcher::vector_element_basic_type(n) == T_DOUBLE));
21963 match(Set dst (VectorCastL2X src));
21964 format %{ "vector_cast_l2x $dst,$src\t!" %}
21965 ins_encode %{
21966 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21967 int vlen = Matcher::vector_length_in_bytes(this, $src);
21968 int vlen_enc = vector_length_encoding(this, $src);
21969 switch (to_elem_bt) {
21970 case T_BYTE:
21971 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
21972 vlen_enc = Assembler::AVX_512bit;
21973 }
21974 __ evpmovqb($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21975 break;
21976 case T_SHORT:
21977 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
21978 vlen_enc = Assembler::AVX_512bit;
21979 }
21980 __ evpmovqw($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21981 break;
21982 case T_INT:
21983 if (vlen == 8) {
21984 if ($dst$$XMMRegister != $src$$XMMRegister) {
21985 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
21986 }
21987 } else if (vlen == 16) {
21988 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 8);
21989 } else if (vlen == 32) {
21990 if (UseAVX > 2) {
21991 if (!VM_Version::supports_avx512vl()) {
21992 vlen_enc = Assembler::AVX_512bit;
21993 }
21994 __ evpmovqd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21995 } else {
21996 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 8, vlen_enc);
21997 __ vpermpd($dst$$XMMRegister, $dst$$XMMRegister, 8, vlen_enc);
21998 }
21999 } else { // vlen == 64
22000 __ evpmovqd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22001 }
22002 break;
22003 case T_FLOAT:
22004 assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "required");
22005 __ evcvtqq2ps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22006 break;
22007 case T_DOUBLE:
22008 assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "required");
22009 __ evcvtqq2pd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22010 break;
22011
22012 default: assert(false, "%s", type2name(to_elem_bt));
22013 }
22014 %}
22015 ins_pipe( pipe_slow );
22016 %}
22017
22018 instruct vcastFtoD_reg(vec dst, vec src) %{
22019 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
22020 match(Set dst (VectorCastF2X src));
22021 format %{ "vector_cast_f2d $dst,$src\t!" %}
22022 ins_encode %{
22023 int vlen_enc = vector_length_encoding(this);
22024 __ vcvtps2pd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22025 %}
22026 ins_pipe( pipe_slow );
22027 %}
22028
22029
22030 instruct castFtoX_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, rFlagsReg cr) %{
22031 predicate(!VM_Version::supports_avx10_2() &&
22032 !VM_Version::supports_avx512vl() &&
22033 Matcher::vector_length_in_bytes(n->in(1)) < 64 &&
22034 type2aelembytes(Matcher::vector_element_basic_type(n)) <= 4 &&
22035 is_integral_type(Matcher::vector_element_basic_type(n)));
22036 match(Set dst (VectorCastF2X src));
22037 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, KILL cr);
22038 format %{ "vector_cast_f2x $dst,$src\t! using $xtmp1, $xtmp2, $xtmp3 and $xtmp4 as TEMP" %}
22039 ins_encode %{
22040 int vlen_enc = vector_length_encoding(this, $src);
22041 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
22042 // JDK-8292878 removed the need for an explicit scratch register needed to load greater than
22043 // 32 bit addresses for register indirect addressing mode since stub constants
22044 // are part of code cache and there is a cap of 2G on ReservedCodeCacheSize currently.
22045 // However, targets are free to increase this limit, but having a large code cache size
22046 // greater than 2G looks unreasonable in practical scenario, on the hind side with given
22047 // cap we save a temporary register allocation which in limiting case can prevent
22048 // spilling in high register pressure blocks.
22049 __ vector_castF2X_avx(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
22050 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister,
22051 ExternalAddress(vector_float_signflip()), noreg, vlen_enc);
22052 %}
22053 ins_pipe( pipe_slow );
22054 %}
22055
22056 instruct castFtoX_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
22057 predicate(!VM_Version::supports_avx10_2() &&
22058 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n->in(1)) == 64) &&
22059 is_integral_type(Matcher::vector_element_basic_type(n)));
22060 match(Set dst (VectorCastF2X src));
22061 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
22062 format %{ "vector_cast_f2x $dst,$src\t! using $xtmp1, $xtmp2, $ktmp1 and $ktmp2 as TEMP" %}
22063 ins_encode %{
22064 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
22065 if (to_elem_bt == T_LONG) {
22066 int vlen_enc = vector_length_encoding(this);
22067 __ vector_castF2L_evex($dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
22068 $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister,
22069 ExternalAddress(vector_double_signflip()), noreg, vlen_enc);
22070 } else {
22071 int vlen_enc = vector_length_encoding(this, $src);
22072 __ vector_castF2X_evex(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
22073 $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister,
22074 ExternalAddress(vector_float_signflip()), noreg, vlen_enc);
22075 }
22076 %}
22077 ins_pipe( pipe_slow );
22078 %}
22079
22080 instruct castFtoX_reg_avx10_2(vec dst, vec src) %{
22081 predicate(VM_Version::supports_avx10_2() &&
22082 is_integral_type(Matcher::vector_element_basic_type(n)));
22083 match(Set dst (VectorCastF2X src));
22084 format %{ "vector_cast_f2x_avx10_2 $dst, $src\t!" %}
22085 ins_encode %{
22086 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
22087 int vlen_enc = (to_elem_bt == T_LONG) ? vector_length_encoding(this) : vector_length_encoding(this, $src);
22088 __ vector_castF2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22089 %}
22090 ins_pipe( pipe_slow );
22091 %}
22092
22093 instruct castFtoX_mem_avx10_2(vec dst, memory src) %{
22094 predicate(VM_Version::supports_avx10_2() &&
22095 is_integral_type(Matcher::vector_element_basic_type(n)));
22096 match(Set dst (VectorCastF2X (LoadVector src)));
22097 format %{ "vector_cast_f2x_avx10_2 $dst, $src\t!" %}
22098 ins_encode %{
22099 int vlen = Matcher::vector_length(this);
22100 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
22101 int vlen_enc = (to_elem_bt == T_LONG) ? vector_length_encoding(this) : vector_length_encoding(vlen * sizeof(jfloat));
22102 __ vector_castF2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$Address, vlen_enc);
22103 %}
22104 ins_pipe( pipe_slow );
22105 %}
22106
22107 instruct vcastDtoF_reg(vec dst, vec src) %{
22108 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
22109 match(Set dst (VectorCastD2X src));
22110 format %{ "vector_cast_d2x $dst,$src\t!" %}
22111 ins_encode %{
22112 int vlen_enc = vector_length_encoding(this, $src);
22113 __ vcvtpd2ps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22114 %}
22115 ins_pipe( pipe_slow );
22116 %}
22117
22118 instruct castDtoX_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, vec xtmp5, rFlagsReg cr) %{
22119 predicate(!VM_Version::supports_avx10_2() &&
22120 !VM_Version::supports_avx512vl() &&
22121 Matcher::vector_length_in_bytes(n->in(1)) < 64 &&
22122 is_integral_type(Matcher::vector_element_basic_type(n)));
22123 match(Set dst (VectorCastD2X src));
22124 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, TEMP xtmp5, KILL cr);
22125 format %{ "vector_cast_d2x $dst,$src\t! using $xtmp1, $xtmp2, $xtmp3, $xtmp4 and $xtmp5 as TEMP" %}
22126 ins_encode %{
22127 int vlen_enc = vector_length_encoding(this, $src);
22128 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
22129 __ vector_castD2X_avx(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
22130 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister, $xtmp5$$XMMRegister,
22131 ExternalAddress(vector_float_signflip()), noreg, vlen_enc);
22132 %}
22133 ins_pipe( pipe_slow );
22134 %}
22135
22136 instruct castDtoX_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
22137 predicate(!VM_Version::supports_avx10_2() &&
22138 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n->in(1)) == 64) &&
22139 is_integral_type(Matcher::vector_element_basic_type(n)));
22140 match(Set dst (VectorCastD2X src));
22141 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
22142 format %{ "vector_cast_d2x $dst,$src\t! using $xtmp1, $xtmp2, $ktmp1 and $ktmp2 as TEMP" %}
22143 ins_encode %{
22144 int vlen_enc = vector_length_encoding(this, $src);
22145 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
22146 AddressLiteral signflip = VM_Version::supports_avx512dq() ? ExternalAddress(vector_double_signflip()) :
22147 ExternalAddress(vector_float_signflip());
22148 __ vector_castD2X_evex(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
22149 $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister, signflip, noreg, vlen_enc);
22150 %}
22151 ins_pipe( pipe_slow );
22152 %}
22153
22154 instruct castDtoX_reg_avx10_2(vec dst, vec src) %{
22155 predicate(VM_Version::supports_avx10_2() &&
22156 is_integral_type(Matcher::vector_element_basic_type(n)));
22157 match(Set dst (VectorCastD2X src));
22158 format %{ "vector_cast_d2x_avx10_2 $dst, $src\t!" %}
22159 ins_encode %{
22160 int vlen_enc = vector_length_encoding(this, $src);
22161 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
22162 __ vector_castD2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22163 %}
22164 ins_pipe( pipe_slow );
22165 %}
22166
22167 instruct castDtoX_mem_avx10_2(vec dst, memory src) %{
22168 predicate(VM_Version::supports_avx10_2() &&
22169 is_integral_type(Matcher::vector_element_basic_type(n)));
22170 match(Set dst (VectorCastD2X (LoadVector src)));
22171 format %{ "vector_cast_d2x_avx10_2 $dst, $src\t!" %}
22172 ins_encode %{
22173 int vlen = Matcher::vector_length(this);
22174 int vlen_enc = vector_length_encoding(vlen * sizeof(jdouble));
22175 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
22176 __ vector_castD2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$Address, vlen_enc);
22177 %}
22178 ins_pipe( pipe_slow );
22179 %}
22180
22181 instruct vucast(vec dst, vec src) %{
22182 match(Set dst (VectorUCastB2X src));
22183 match(Set dst (VectorUCastS2X src));
22184 match(Set dst (VectorUCastI2X src));
22185 format %{ "vector_ucast $dst,$src\t!" %}
22186 ins_encode %{
22187 assert(UseAVX > 0, "required");
22188
22189 BasicType from_elem_bt = Matcher::vector_element_basic_type(this, $src);
22190 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
22191 int vlen_enc = vector_length_encoding(this);
22192 __ vector_unsigned_cast($dst$$XMMRegister, $src$$XMMRegister, vlen_enc, from_elem_bt, to_elem_bt);
22193 %}
22194 ins_pipe( pipe_slow );
22195 %}
22196
22197 instruct vround_float_avx(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, rFlagsReg cr) %{
22198 predicate(!VM_Version::supports_avx512vl() &&
22199 Matcher::vector_length_in_bytes(n) < 64 &&
22200 Matcher::vector_element_basic_type(n) == T_INT);
22201 match(Set dst (RoundVF src));
22202 effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, KILL cr);
22203 format %{ "vector_round_float $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $xtmp3, $xtmp4 as TEMP" %}
22204 ins_encode %{
22205 int vlen_enc = vector_length_encoding(this);
22206 InternalAddress new_mxcsr = $constantaddress((jint)(EnableX86ECoreOpts ? 0x3FBF : 0x3F80));
22207 __ vector_round_float_avx($dst$$XMMRegister, $src$$XMMRegister,
22208 ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), new_mxcsr, vlen_enc,
22209 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister);
22210 %}
22211 ins_pipe( pipe_slow );
22212 %}
22213
22214 instruct vround_float_evex(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
22215 predicate((VM_Version::supports_avx512vl() ||
22216 Matcher::vector_length_in_bytes(n) == 64) &&
22217 Matcher::vector_element_basic_type(n) == T_INT);
22218 match(Set dst (RoundVF src));
22219 effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
22220 format %{ "vector_round_float $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $ktmp1, $ktmp2 as TEMP" %}
22221 ins_encode %{
22222 int vlen_enc = vector_length_encoding(this);
22223 InternalAddress new_mxcsr = $constantaddress((jint)(EnableX86ECoreOpts ? 0x3FBF : 0x3F80));
22224 __ vector_round_float_evex($dst$$XMMRegister, $src$$XMMRegister,
22225 ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), new_mxcsr, vlen_enc,
22226 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
22227 %}
22228 ins_pipe( pipe_slow );
22229 %}
22230
22231 instruct vround_reg_evex(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
22232 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
22233 match(Set dst (RoundVD src));
22234 effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
22235 format %{ "vector_round_long $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $ktmp1, $ktmp2 as TEMP" %}
22236 ins_encode %{
22237 int vlen_enc = vector_length_encoding(this);
22238 InternalAddress new_mxcsr = $constantaddress((jint)(EnableX86ECoreOpts ? 0x3FBF : 0x3F80));
22239 __ vector_round_double_evex($dst$$XMMRegister, $src$$XMMRegister,
22240 ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), new_mxcsr, vlen_enc,
22241 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
22242 %}
22243 ins_pipe( pipe_slow );
22244 %}
22245
22246 // --------------------------------- VectorMaskCmp --------------------------------------
22247
22248 instruct vcmpFD(legVec dst, legVec src1, legVec src2, immI8 cond) %{
22249 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
22250 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 8 && // src1
22251 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
22252 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
22253 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22254 format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
22255 ins_encode %{
22256 int vlen_enc = vector_length_encoding(this, $src1);
22257 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
22258 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
22259 __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
22260 } else {
22261 __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
22262 }
22263 %}
22264 ins_pipe( pipe_slow );
22265 %}
22266
22267 instruct evcmpFD64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
22268 predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64 && // src1
22269 n->bottom_type()->isa_pvectmask() == nullptr &&
22270 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
22271 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22272 effect(TEMP ktmp);
22273 format %{ "vector_compare $dst,$src1,$src2,$cond" %}
22274 ins_encode %{
22275 int vlen_enc = Assembler::AVX_512bit;
22276 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
22277 KRegister mask = k0; // The comparison itself is not being masked.
22278 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
22279 __ evcmpps($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
22280 __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, noreg);
22281 } else {
22282 __ evcmppd($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
22283 __ evmovdquq($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, noreg);
22284 }
22285 %}
22286 ins_pipe( pipe_slow );
22287 %}
22288
22289 instruct evcmpFD(kReg dst, vec src1, vec src2, immI8 cond) %{
22290 predicate(n->bottom_type()->isa_pvectmask() &&
22291 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
22292 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22293 format %{ "vector_compare_evex $dst,$src1,$src2,$cond\t!" %}
22294 ins_encode %{
22295 assert(bottom_type()->isa_pvectmask(), "TypePVectMask expected");
22296 int vlen_enc = vector_length_encoding(this, $src1);
22297 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
22298 KRegister mask = k0; // The comparison itself is not being masked.
22299 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
22300 __ evcmpps($dst$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
22301 } else {
22302 __ evcmppd($dst$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
22303 }
22304 %}
22305 ins_pipe( pipe_slow );
22306 %}
22307
22308 instruct vcmp_direct(legVec dst, legVec src1, legVec src2, immI8 cond) %{
22309 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
22310 !Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
22311 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
22312 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
22313 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1))) &&
22314 (n->in(2)->get_int() == BoolTest::eq ||
22315 n->in(2)->get_int() == BoolTest::lt ||
22316 n->in(2)->get_int() == BoolTest::gt)); // cond
22317 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22318 format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
22319 ins_encode %{
22320 int vlen_enc = vector_length_encoding(this, $src1);
22321 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22322 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
22323 __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, cmp, ww, vlen_enc);
22324 %}
22325 ins_pipe( pipe_slow );
22326 %}
22327
22328 instruct vcmp_negate(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
22329 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
22330 !Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
22331 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
22332 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
22333 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1))) &&
22334 (n->in(2)->get_int() == BoolTest::ne ||
22335 n->in(2)->get_int() == BoolTest::le ||
22336 n->in(2)->get_int() == BoolTest::ge)); // cond
22337 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22338 effect(TEMP dst, TEMP xtmp);
22339 format %{ "vector_compare $dst,$src1,$src2,$cond\t! using $xtmp as TEMP" %}
22340 ins_encode %{
22341 int vlen_enc = vector_length_encoding(this, $src1);
22342 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22343 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
22344 __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $xtmp$$XMMRegister, cmp, ww, vlen_enc);
22345 %}
22346 ins_pipe( pipe_slow );
22347 %}
22348
22349 instruct vcmpu(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
22350 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
22351 Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
22352 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
22353 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
22354 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
22355 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22356 effect(TEMP dst, TEMP xtmp);
22357 format %{ "vector_compareu $dst,$src1,$src2,$cond\t! using $xtmp as TEMP" %}
22358 ins_encode %{
22359 InternalAddress flip_bit = $constantaddress(high_bit_set(Matcher::vector_element_basic_type(this, $src1)));
22360 int vlen_enc = vector_length_encoding(this, $src1);
22361 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22362 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
22363
22364 if (vlen_enc == Assembler::AVX_128bit) {
22365 __ vmovddup($xtmp$$XMMRegister, flip_bit, vlen_enc, noreg);
22366 } else {
22367 __ vbroadcastsd($xtmp$$XMMRegister, flip_bit, vlen_enc, noreg);
22368 }
22369 __ vpxor($dst$$XMMRegister, $xtmp$$XMMRegister, $src1$$XMMRegister, vlen_enc);
22370 __ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $src2$$XMMRegister, vlen_enc);
22371 __ vpcmpCCW($dst$$XMMRegister, $dst$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, cmp, ww, vlen_enc);
22372 %}
22373 ins_pipe( pipe_slow );
22374 %}
22375
22376 instruct vcmp64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
22377 predicate((n->bottom_type()->isa_pvectmask() == nullptr &&
22378 Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64) && // src1
22379 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
22380 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22381 effect(TEMP ktmp);
22382 format %{ "vector_compare $dst,$src1,$src2,$cond" %}
22383 ins_encode %{
22384 assert(UseAVX > 2, "required");
22385
22386 int vlen_enc = vector_length_encoding(this, $src1);
22387 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22388 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
22389 KRegister mask = k0; // The comparison itself is not being masked.
22390 bool merge = false;
22391 BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
22392
22393 switch (src1_elem_bt) {
22394 case T_INT: {
22395 __ evpcmpd($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22396 __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), merge, vlen_enc, noreg);
22397 break;
22398 }
22399 case T_LONG: {
22400 __ evpcmpq($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22401 __ evmovdquq($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), merge, vlen_enc, noreg);
22402 break;
22403 }
22404 default: assert(false, "%s", type2name(src1_elem_bt));
22405 }
22406 %}
22407 ins_pipe( pipe_slow );
22408 %}
22409
22410
22411 instruct evcmp(kReg dst, vec src1, vec src2, immI8 cond) %{
22412 predicate(n->bottom_type()->isa_pvectmask() &&
22413 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
22414 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22415 format %{ "vector_compared_evex $dst,$src1,$src2,$cond\t!" %}
22416 ins_encode %{
22417 assert(UseAVX > 2, "required");
22418 assert(bottom_type()->isa_pvectmask(), "TypePVectMask expected");
22419
22420 int vlen_enc = vector_length_encoding(this, $src1);
22421 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22422 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
22423 BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
22424
22425 // Comparison i
22426 switch (src1_elem_bt) {
22427 case T_BYTE: {
22428 __ evpcmpb($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22429 break;
22430 }
22431 case T_SHORT: {
22432 __ evpcmpw($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22433 break;
22434 }
22435 case T_INT: {
22436 __ evpcmpd($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22437 break;
22438 }
22439 case T_LONG: {
22440 __ evpcmpq($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22441 break;
22442 }
22443 default: assert(false, "%s", type2name(src1_elem_bt));
22444 }
22445 %}
22446 ins_pipe( pipe_slow );
22447 %}
22448
22449 // Extract
22450
22451 instruct extractI(rRegI dst, legVec src, immU8 idx) %{
22452 predicate(Matcher::vector_length_in_bytes(n->in(1)) <= 16); // src
22453 match(Set dst (ExtractI src idx));
22454 match(Set dst (ExtractS src idx));
22455 match(Set dst (ExtractB src idx));
22456 format %{ "extractI $dst,$src,$idx\t!" %}
22457 ins_encode %{
22458 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22459
22460 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
22461 __ get_elem(elem_bt, $dst$$Register, $src$$XMMRegister, $idx$$constant);
22462 %}
22463 ins_pipe( pipe_slow );
22464 %}
22465
22466 instruct vextractI(rRegI dst, legVec src, immI idx, legVec vtmp) %{
22467 predicate(Matcher::vector_length_in_bytes(n->in(1)) == 32 || // src
22468 Matcher::vector_length_in_bytes(n->in(1)) == 64); // src
22469 match(Set dst (ExtractI src idx));
22470 match(Set dst (ExtractS src idx));
22471 match(Set dst (ExtractB src idx));
22472 effect(TEMP vtmp);
22473 format %{ "vextractI $dst,$src,$idx\t! using $vtmp as TEMP" %}
22474 ins_encode %{
22475 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22476
22477 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
22478 XMMRegister lane_xmm = __ get_lane(elem_bt, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22479 __ get_elem(elem_bt, $dst$$Register, lane_xmm, $idx$$constant);
22480 %}
22481 ins_pipe( pipe_slow );
22482 %}
22483
22484 instruct extractL(rRegL dst, legVec src, immU8 idx) %{
22485 predicate(Matcher::vector_length(n->in(1)) <= 2); // src
22486 match(Set dst (ExtractL src idx));
22487 format %{ "extractL $dst,$src,$idx\t!" %}
22488 ins_encode %{
22489 assert(UseSSE >= 4, "required");
22490 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22491
22492 __ get_elem(T_LONG, $dst$$Register, $src$$XMMRegister, $idx$$constant);
22493 %}
22494 ins_pipe( pipe_slow );
22495 %}
22496
22497 instruct vextractL(rRegL dst, legVec src, immU8 idx, legVec vtmp) %{
22498 predicate(Matcher::vector_length(n->in(1)) == 4 || // src
22499 Matcher::vector_length(n->in(1)) == 8); // src
22500 match(Set dst (ExtractL src idx));
22501 effect(TEMP vtmp);
22502 format %{ "vextractL $dst,$src,$idx\t! using $vtmp as TEMP" %}
22503 ins_encode %{
22504 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22505
22506 XMMRegister lane_reg = __ get_lane(T_LONG, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22507 __ get_elem(T_LONG, $dst$$Register, lane_reg, $idx$$constant);
22508 %}
22509 ins_pipe( pipe_slow );
22510 %}
22511
22512 instruct extractF(legRegF dst, legVec src, immU8 idx, legVec vtmp) %{
22513 predicate(Matcher::vector_length(n->in(1)) <= 4);
22514 match(Set dst (ExtractF src idx));
22515 effect(TEMP dst, TEMP vtmp);
22516 format %{ "extractF $dst,$src,$idx\t! using $vtmp as TEMP" %}
22517 ins_encode %{
22518 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22519
22520 __ get_elem(T_FLOAT, $dst$$XMMRegister, $src$$XMMRegister, $idx$$constant, $vtmp$$XMMRegister);
22521 %}
22522 ins_pipe( pipe_slow );
22523 %}
22524
22525 instruct vextractF(legRegF dst, legVec src, immU8 idx, legVec vtmp) %{
22526 predicate(Matcher::vector_length(n->in(1)/*src*/) == 8 ||
22527 Matcher::vector_length(n->in(1)/*src*/) == 16);
22528 match(Set dst (ExtractF src idx));
22529 effect(TEMP vtmp);
22530 format %{ "vextractF $dst,$src,$idx\t! using $vtmp as TEMP" %}
22531 ins_encode %{
22532 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22533
22534 XMMRegister lane_reg = __ get_lane(T_FLOAT, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22535 __ get_elem(T_FLOAT, $dst$$XMMRegister, lane_reg, $idx$$constant);
22536 %}
22537 ins_pipe( pipe_slow );
22538 %}
22539
22540 instruct extractD(legRegD dst, legVec src, immU8 idx) %{
22541 predicate(Matcher::vector_length(n->in(1)) == 2); // src
22542 match(Set dst (ExtractD src idx));
22543 format %{ "extractD $dst,$src,$idx\t!" %}
22544 ins_encode %{
22545 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22546
22547 __ get_elem(T_DOUBLE, $dst$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22548 %}
22549 ins_pipe( pipe_slow );
22550 %}
22551
22552 instruct vextractD(legRegD dst, legVec src, immU8 idx, legVec vtmp) %{
22553 predicate(Matcher::vector_length(n->in(1)) == 4 || // src
22554 Matcher::vector_length(n->in(1)) == 8); // src
22555 match(Set dst (ExtractD src idx));
22556 effect(TEMP vtmp);
22557 format %{ "vextractD $dst,$src,$idx\t! using $vtmp as TEMP" %}
22558 ins_encode %{
22559 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22560
22561 XMMRegister lane_reg = __ get_lane(T_DOUBLE, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22562 __ get_elem(T_DOUBLE, $dst$$XMMRegister, lane_reg, $idx$$constant);
22563 %}
22564 ins_pipe( pipe_slow );
22565 %}
22566
22567 // --------------------------------- Vector Blend --------------------------------------
22568
22569 instruct blendvp(vec dst, vec src, vec mask, rxmm0 tmp) %{
22570 predicate(UseAVX == 0);
22571 match(Set dst (VectorBlend (Binary dst src) mask));
22572 format %{ "vector_blend $dst,$src,$mask\t! using $tmp as TEMP" %}
22573 effect(TEMP tmp);
22574 ins_encode %{
22575 assert(UseSSE >= 4, "required");
22576
22577 if ($mask$$XMMRegister != $tmp$$XMMRegister) {
22578 __ movdqu($tmp$$XMMRegister, $mask$$XMMRegister);
22579 }
22580 __ pblendvb($dst$$XMMRegister, $src$$XMMRegister); // uses xmm0 as mask
22581 %}
22582 ins_pipe( pipe_slow );
22583 %}
22584
22585 instruct vblendvpI(legVec dst, legVec src1, legVec src2, legVec mask) %{
22586 predicate(UseAVX > 0 && !EnableX86ECoreOpts &&
22587 n->in(2)->bottom_type()->isa_pvectmask() == nullptr &&
22588 Matcher::vector_length_in_bytes(n) <= 32 &&
22589 is_integral_type(Matcher::vector_element_basic_type(n)));
22590 match(Set dst (VectorBlend (Binary src1 src2) mask));
22591 format %{ "vector_blend $dst,$src1,$src2,$mask\t!" %}
22592 ins_encode %{
22593 int vlen_enc = vector_length_encoding(this);
22594 __ vpblendvb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $mask$$XMMRegister, vlen_enc);
22595 %}
22596 ins_pipe( pipe_slow );
22597 %}
22598
22599 instruct vblendvpFD(legVec dst, legVec src1, legVec src2, legVec mask) %{
22600 predicate(UseAVX > 0 && !EnableX86ECoreOpts &&
22601 n->in(2)->bottom_type()->isa_pvectmask() == nullptr &&
22602 Matcher::vector_length_in_bytes(n) <= 32 &&
22603 !is_integral_type(Matcher::vector_element_basic_type(n)));
22604 match(Set dst (VectorBlend (Binary src1 src2) mask));
22605 format %{ "vector_blend $dst,$src1,$src2,$mask\t!" %}
22606 ins_encode %{
22607 int vlen_enc = vector_length_encoding(this);
22608 __ vblendvps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $mask$$XMMRegister, vlen_enc);
22609 %}
22610 ins_pipe( pipe_slow );
22611 %}
22612
22613 instruct vblendvp(legVec dst, legVec src1, legVec src2, legVec mask, legVec vtmp) %{
22614 predicate(UseAVX > 0 && EnableX86ECoreOpts &&
22615 n->in(2)->bottom_type()->isa_pvectmask() == nullptr &&
22616 Matcher::vector_length_in_bytes(n) <= 32);
22617 match(Set dst (VectorBlend (Binary src1 src2) mask));
22618 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using $vtmp as TEMP" %}
22619 effect(TEMP vtmp, TEMP dst);
22620 ins_encode %{
22621 int vlen_enc = vector_length_encoding(this);
22622 __ vpandn($vtmp$$XMMRegister, $mask$$XMMRegister, $src1$$XMMRegister, vlen_enc);
22623 __ vpand ($dst$$XMMRegister, $mask$$XMMRegister, $src2$$XMMRegister, vlen_enc);
22624 __ vpor ($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22625 %}
22626 ins_pipe( pipe_slow );
22627 %}
22628
22629 instruct evblendvp64(vec dst, vec src1, vec src2, vec mask, kReg ktmp) %{
22630 predicate(Matcher::vector_length_in_bytes(n) == 64 &&
22631 n->in(2)->bottom_type()->isa_pvectmask() == nullptr);
22632 match(Set dst (VectorBlend (Binary src1 src2) mask));
22633 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
22634 effect(TEMP ktmp);
22635 ins_encode %{
22636 int vlen_enc = Assembler::AVX_512bit;
22637 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22638 __ evpcmp(elem_bt, $ktmp$$KRegister, k0, $mask$$XMMRegister, ExternalAddress(vector_all_bits_set()), Assembler::eq, vlen_enc, noreg);
22639 __ evpblend(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
22640 %}
22641 ins_pipe( pipe_slow );
22642 %}
22643
22644
22645 instruct evblendvp64_masked(vec dst, vec src1, vec src2, kReg mask) %{
22646 predicate(n->in(2)->bottom_type()->isa_pvectmask() &&
22647 (!is_subword_type(Matcher::vector_element_basic_type(n)) ||
22648 VM_Version::supports_avx512bw()));
22649 match(Set dst (VectorBlend (Binary src1 src2) mask));
22650 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
22651 ins_encode %{
22652 int vlen_enc = vector_length_encoding(this);
22653 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22654 __ evpblend(elem_bt, $dst$$XMMRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
22655 %}
22656 ins_pipe( pipe_slow );
22657 %}
22658
22659 // --------------------------------- ABS --------------------------------------
22660 // a = |a|
22661 instruct vabsB_reg(vec dst, vec src) %{
22662 match(Set dst (AbsVB src));
22663 format %{ "vabsb $dst,$src\t# $dst = |$src| abs packedB" %}
22664 ins_encode %{
22665 uint vlen = Matcher::vector_length(this);
22666 if (vlen <= 16) {
22667 __ pabsb($dst$$XMMRegister, $src$$XMMRegister);
22668 } else {
22669 int vlen_enc = vector_length_encoding(this);
22670 __ vpabsb($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22671 }
22672 %}
22673 ins_pipe( pipe_slow );
22674 %}
22675
22676 instruct vabsS_reg(vec dst, vec src) %{
22677 match(Set dst (AbsVS src));
22678 format %{ "vabsw $dst,$src\t# $dst = |$src| abs packedS" %}
22679 ins_encode %{
22680 uint vlen = Matcher::vector_length(this);
22681 if (vlen <= 8) {
22682 __ pabsw($dst$$XMMRegister, $src$$XMMRegister);
22683 } else {
22684 int vlen_enc = vector_length_encoding(this);
22685 __ vpabsw($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22686 }
22687 %}
22688 ins_pipe( pipe_slow );
22689 %}
22690
22691 instruct vabsI_reg(vec dst, vec src) %{
22692 match(Set dst (AbsVI src));
22693 format %{ "pabsd $dst,$src\t# $dst = |$src| abs packedI" %}
22694 ins_encode %{
22695 uint vlen = Matcher::vector_length(this);
22696 if (vlen <= 4) {
22697 __ pabsd($dst$$XMMRegister, $src$$XMMRegister);
22698 } else {
22699 int vlen_enc = vector_length_encoding(this);
22700 __ vpabsd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22701 }
22702 %}
22703 ins_pipe( pipe_slow );
22704 %}
22705
22706 instruct vabsL_reg(vec dst, vec src) %{
22707 match(Set dst (AbsVL src));
22708 format %{ "evpabsq $dst,$src\t# $dst = |$src| abs packedL" %}
22709 ins_encode %{
22710 assert(UseAVX > 2, "required");
22711 int vlen_enc = vector_length_encoding(this);
22712 if (!VM_Version::supports_avx512vl()) {
22713 vlen_enc = Assembler::AVX_512bit;
22714 }
22715 __ evpabsq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22716 %}
22717 ins_pipe( pipe_slow );
22718 %}
22719
22720 // --------------------------------- ABSNEG --------------------------------------
22721
22722 instruct vabsnegF(vec dst, vec src) %{
22723 predicate(Matcher::vector_length(n) != 4); // handled by 1-operand instruction vabsneg4F
22724 match(Set dst (AbsVF src));
22725 match(Set dst (NegVF src));
22726 format %{ "vabsnegf $dst,$src,[mask]\t# absneg packedF" %}
22727 ins_cost(150);
22728 ins_encode %{
22729 int opcode = this->ideal_Opcode();
22730 int vlen = Matcher::vector_length(this);
22731 if (vlen == 2) {
22732 __ vabsnegf(opcode, $dst$$XMMRegister, $src$$XMMRegister);
22733 } else {
22734 assert(vlen == 8 || vlen == 16, "required");
22735 int vlen_enc = vector_length_encoding(this);
22736 __ vabsnegf(opcode, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22737 }
22738 %}
22739 ins_pipe( pipe_slow );
22740 %}
22741
22742 instruct vabsneg4F(vec dst) %{
22743 predicate(Matcher::vector_length(n) == 4);
22744 match(Set dst (AbsVF dst));
22745 match(Set dst (NegVF dst));
22746 format %{ "vabsnegf $dst,[mask]\t# absneg packed4F" %}
22747 ins_cost(150);
22748 ins_encode %{
22749 int opcode = this->ideal_Opcode();
22750 __ vabsnegf(opcode, $dst$$XMMRegister, $dst$$XMMRegister);
22751 %}
22752 ins_pipe( pipe_slow );
22753 %}
22754
22755 instruct vabsnegD(vec dst, vec src) %{
22756 match(Set dst (AbsVD src));
22757 match(Set dst (NegVD src));
22758 format %{ "vabsnegd $dst,$src,[mask]\t# absneg packedD" %}
22759 ins_encode %{
22760 int opcode = this->ideal_Opcode();
22761 uint vlen = Matcher::vector_length(this);
22762 if (vlen == 2) {
22763 __ vabsnegd(opcode, $dst$$XMMRegister, $src$$XMMRegister);
22764 } else {
22765 int vlen_enc = vector_length_encoding(this);
22766 __ vabsnegd(opcode, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22767 }
22768 %}
22769 ins_pipe( pipe_slow );
22770 %}
22771
22772 //------------------------------------- VectorTest --------------------------------------------
22773
22774 instruct vptest_lt16(rFlagsRegU cr, legVec src1, legVec src2, legVec vtmp) %{
22775 predicate(Matcher::vector_length_in_bytes(n->in(1)) < 16);
22776 match(Set cr (VectorTest src1 src2));
22777 effect(TEMP vtmp);
22778 format %{ "vptest_lt16 $src1, $src2\t! using $vtmp as TEMP" %}
22779 ins_encode %{
22780 BasicType bt = Matcher::vector_element_basic_type(this, $src1);
22781 int vlen = Matcher::vector_length_in_bytes(this, $src1);
22782 __ vectortest(bt, $src1$$XMMRegister, $src2$$XMMRegister, $vtmp$$XMMRegister, vlen);
22783 %}
22784 ins_pipe( pipe_slow );
22785 %}
22786
22787 instruct vptest_ge16(rFlagsRegU cr, legVec src1, legVec src2) %{
22788 predicate(Matcher::vector_length_in_bytes(n->in(1)) >= 16);
22789 match(Set cr (VectorTest src1 src2));
22790 format %{ "vptest_ge16 $src1, $src2\n\t" %}
22791 ins_encode %{
22792 BasicType bt = Matcher::vector_element_basic_type(this, $src1);
22793 int vlen = Matcher::vector_length_in_bytes(this, $src1);
22794 __ vectortest(bt, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, vlen);
22795 %}
22796 ins_pipe( pipe_slow );
22797 %}
22798
22799 instruct ktest_alltrue_le8(rFlagsRegU cr, kReg src1, kReg src2, rRegI tmp) %{
22800 predicate((Matcher::vector_length(n->in(1)) < 8 ||
22801 (Matcher::vector_length(n->in(1)) == 8 && !VM_Version::supports_avx512dq())) &&
22802 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
22803 match(Set cr (VectorTest src1 src2));
22804 effect(TEMP tmp);
22805 format %{ "ktest_alltrue_le8 $src1, $src2\t! using $tmp as TEMP" %}
22806 ins_encode %{
22807 uint masklen = Matcher::vector_length(this, $src1);
22808 __ kmovwl($tmp$$Register, $src1$$KRegister);
22809 __ andl($tmp$$Register, (1 << masklen) - 1);
22810 __ cmpl($tmp$$Register, (1 << masklen) - 1);
22811 %}
22812 ins_pipe( pipe_slow );
22813 %}
22814
22815 instruct ktest_anytrue_le8(rFlagsRegU cr, kReg src1, kReg src2, rRegI tmp) %{
22816 predicate((Matcher::vector_length(n->in(1)) < 8 ||
22817 (Matcher::vector_length(n->in(1)) == 8 && !VM_Version::supports_avx512dq())) &&
22818 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
22819 match(Set cr (VectorTest src1 src2));
22820 effect(TEMP tmp);
22821 format %{ "ktest_anytrue_le8 $src1, $src2\t! using $tmp as TEMP" %}
22822 ins_encode %{
22823 uint masklen = Matcher::vector_length(this, $src1);
22824 __ kmovwl($tmp$$Register, $src1$$KRegister);
22825 __ andl($tmp$$Register, (1 << masklen) - 1);
22826 %}
22827 ins_pipe( pipe_slow );
22828 %}
22829
22830 instruct ktest_ge8(rFlagsRegU cr, kReg src1, kReg src2) %{
22831 predicate(Matcher::vector_length(n->in(1)) >= 16 ||
22832 (Matcher::vector_length(n->in(1)) == 8 && VM_Version::supports_avx512dq()));
22833 match(Set cr (VectorTest src1 src2));
22834 format %{ "ktest_ge8 $src1, $src2\n\t" %}
22835 ins_encode %{
22836 uint masklen = Matcher::vector_length(this, $src1);
22837 __ kortest(masklen, $src1$$KRegister, $src1$$KRegister);
22838 %}
22839 ins_pipe( pipe_slow );
22840 %}
22841
22842 //------------------------------------- LoadMask --------------------------------------------
22843
22844 instruct loadMask(legVec dst, legVec src) %{
22845 predicate(n->bottom_type()->isa_pvectmask() == nullptr && !VM_Version::supports_avx512vlbw());
22846 match(Set dst (VectorLoadMask src));
22847 effect(TEMP dst);
22848 format %{ "vector_loadmask_byte $dst, $src\n\t" %}
22849 ins_encode %{
22850 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
22851 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22852 __ load_vector_mask($dst$$XMMRegister, $src$$XMMRegister, vlen_in_bytes, elem_bt, true);
22853 %}
22854 ins_pipe( pipe_slow );
22855 %}
22856
22857 instruct loadMask64(kReg dst, vec src, vec xtmp) %{
22858 predicate(n->bottom_type()->isa_pvectmask() && !VM_Version::supports_avx512vlbw());
22859 match(Set dst (VectorLoadMask src));
22860 effect(TEMP xtmp);
22861 format %{ "vector_loadmask_64byte $dst, $src\t! using $xtmp as TEMP" %}
22862 ins_encode %{
22863 __ load_vector_mask($dst$$KRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
22864 true, Assembler::AVX_512bit);
22865 %}
22866 ins_pipe( pipe_slow );
22867 %}
22868
22869 instruct loadMask_evex(kReg dst, vec src, vec xtmp) %{
22870 predicate(n->bottom_type()->isa_pvectmask() && VM_Version::supports_avx512vlbw());
22871 match(Set dst (VectorLoadMask src));
22872 effect(TEMP xtmp);
22873 format %{ "vector_loadmask_byte $dst, $src\t! using $xtmp as TEMP" %}
22874 ins_encode %{
22875 int vlen_enc = vector_length_encoding(in(1));
22876 __ load_vector_mask($dst$$KRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
22877 false, vlen_enc);
22878 %}
22879 ins_pipe( pipe_slow );
22880 %}
22881
22882 //------------------------------------- StoreMask --------------------------------------------
22883
22884 instruct vstoreMask1B(vec dst, vec src, immI_1 size) %{
22885 predicate(Matcher::vector_length(n) < 64 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22886 match(Set dst (VectorStoreMask src size));
22887 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22888 ins_encode %{
22889 int vlen = Matcher::vector_length(this);
22890 if (vlen <= 16 && UseAVX <= 2) {
22891 assert(UseSSE >= 3, "required");
22892 __ pabsb($dst$$XMMRegister, $src$$XMMRegister);
22893 } else {
22894 assert(UseAVX > 0, "required");
22895 int src_vlen_enc = vector_length_encoding(this, $src);
22896 __ vpabsb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
22897 }
22898 %}
22899 ins_pipe( pipe_slow );
22900 %}
22901
22902 instruct vstoreMask2B(vec dst, vec src, vec xtmp, immI_2 size) %{
22903 predicate(Matcher::vector_length(n) <= 16 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22904 match(Set dst (VectorStoreMask src size));
22905 effect(TEMP_DEF dst, TEMP xtmp);
22906 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22907 ins_encode %{
22908 int vlen_enc = Assembler::AVX_128bit;
22909 int vlen = Matcher::vector_length(this);
22910 if (vlen <= 8) {
22911 assert(UseSSE >= 3, "required");
22912 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
22913 __ pabsw($dst$$XMMRegister, $src$$XMMRegister);
22914 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
22915 } else {
22916 assert(UseAVX > 0, "required");
22917 __ vextracti128($dst$$XMMRegister, $src$$XMMRegister, 0x1);
22918 __ vpacksswb($dst$$XMMRegister, $src$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22919 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22920 }
22921 %}
22922 ins_pipe( pipe_slow );
22923 %}
22924
22925 instruct vstoreMask4B(vec dst, vec src, vec xtmp, immI_4 size) %{
22926 predicate(UseAVX <= 2 && Matcher::vector_length(n) <= 8 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22927 match(Set dst (VectorStoreMask src size));
22928 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22929 effect(TEMP_DEF dst, TEMP xtmp);
22930 ins_encode %{
22931 int vlen_enc = Assembler::AVX_128bit;
22932 int vlen = Matcher::vector_length(this);
22933 if (vlen <= 4) {
22934 assert(UseSSE >= 3, "required");
22935 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
22936 __ pabsd($dst$$XMMRegister, $src$$XMMRegister);
22937 __ packusdw($dst$$XMMRegister, $xtmp$$XMMRegister);
22938 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
22939 } else {
22940 assert(UseAVX > 0, "required");
22941 __ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
22942 __ vextracti128($dst$$XMMRegister, $src$$XMMRegister, 0x1);
22943 __ vpackssdw($dst$$XMMRegister, $src$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22944 __ vpacksswb($dst$$XMMRegister, $dst$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
22945 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22946 }
22947 %}
22948 ins_pipe( pipe_slow );
22949 %}
22950
22951 instruct storeMask8B(vec dst, vec src, vec xtmp, immI_8 size) %{
22952 predicate(UseAVX <= 2 && Matcher::vector_length(n) == 2);
22953 match(Set dst (VectorStoreMask src size));
22954 effect(TEMP_DEF dst, TEMP xtmp);
22955 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22956 ins_encode %{
22957 assert(UseSSE >= 3, "required");
22958 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
22959 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x8);
22960 __ pabsd($dst$$XMMRegister, $dst$$XMMRegister);
22961 __ packusdw($dst$$XMMRegister, $xtmp$$XMMRegister);
22962 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
22963 %}
22964 ins_pipe( pipe_slow );
22965 %}
22966
22967 instruct storeMask8B_avx(vec dst, vec src, immI_8 size, vec vtmp) %{
22968 predicate(UseAVX <= 2 && Matcher::vector_length(n) == 4);
22969 match(Set dst (VectorStoreMask src size));
22970 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s], using $vtmp as TEMP" %}
22971 effect(TEMP_DEF dst, TEMP vtmp);
22972 ins_encode %{
22973 int vlen_enc = Assembler::AVX_128bit;
22974 __ vshufps($dst$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 0x88, Assembler::AVX_256bit);
22975 __ vextracti128($vtmp$$XMMRegister, $dst$$XMMRegister, 0x1);
22976 __ vblendps($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0xC, vlen_enc);
22977 __ vpxor($vtmp$$XMMRegister, $vtmp$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22978 __ vpackssdw($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22979 __ vpacksswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22980 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22981 %}
22982 ins_pipe( pipe_slow );
22983 %}
22984
22985 instruct vstoreMask4B_evex_novectmask(vec dst, vec src, immI_4 size) %{
22986 predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22987 match(Set dst (VectorStoreMask src size));
22988 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22989 ins_encode %{
22990 int src_vlen_enc = vector_length_encoding(this, $src);
22991 int dst_vlen_enc = vector_length_encoding(this);
22992 if (!VM_Version::supports_avx512vl()) {
22993 src_vlen_enc = Assembler::AVX_512bit;
22994 }
22995 __ evpmovdb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
22996 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
22997 %}
22998 ins_pipe( pipe_slow );
22999 %}
23000
23001 instruct vstoreMask8B_evex_novectmask(vec dst, vec src, immI_8 size) %{
23002 predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23003 match(Set dst (VectorStoreMask src size));
23004 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
23005 ins_encode %{
23006 int src_vlen_enc = vector_length_encoding(this, $src);
23007 int dst_vlen_enc = vector_length_encoding(this);
23008 if (!VM_Version::supports_avx512vl()) {
23009 src_vlen_enc = Assembler::AVX_512bit;
23010 }
23011 __ evpmovqb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
23012 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
23013 %}
23014 ins_pipe( pipe_slow );
23015 %}
23016
23017 instruct vstoreMask_evex_vectmask(vec dst, kReg mask, immI size) %{
23018 predicate(n->in(1)->bottom_type()->isa_pvectmask() && !VM_Version::supports_avx512vlbw());
23019 match(Set dst (VectorStoreMask mask size));
23020 effect(TEMP_DEF dst);
23021 format %{ "vector_store_mask $dst, $mask \t! elem size is $size byte[s]" %}
23022 ins_encode %{
23023 assert(Matcher::vector_length_in_bytes(this, $mask) == 64, "");
23024 __ evmovdqul($dst$$XMMRegister, $mask$$KRegister, ExternalAddress(vector_int_mask_cmp_bits()),
23025 false, Assembler::AVX_512bit, noreg);
23026 __ evpmovdb($dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_512bit);
23027 %}
23028 ins_pipe( pipe_slow );
23029 %}
23030
23031 instruct vstoreMask_evex(vec dst, kReg mask, immI size) %{
23032 predicate(n->in(1)->bottom_type()->isa_pvectmask() && VM_Version::supports_avx512vlbw());
23033 match(Set dst (VectorStoreMask mask size));
23034 effect(TEMP_DEF dst);
23035 format %{ "vector_store_mask $dst, $mask \t! elem size is $size byte[s]" %}
23036 ins_encode %{
23037 int dst_vlen_enc = vector_length_encoding(this);
23038 __ evpmovm2b($dst$$XMMRegister, $mask$$KRegister, dst_vlen_enc);
23039 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
23040 %}
23041 ins_pipe( pipe_slow );
23042 %}
23043
23044 instruct vmaskcast_evex(kReg dst) %{
23045 match(Set dst (VectorMaskCast dst));
23046 ins_cost(0);
23047 format %{ "vector_mask_cast $dst" %}
23048 ins_encode %{
23049 // empty
23050 %}
23051 ins_pipe(empty);
23052 %}
23053
23054 instruct vmaskcast(vec dst) %{
23055 predicate(Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
23056 match(Set dst (VectorMaskCast dst));
23057 ins_cost(0);
23058 format %{ "vector_mask_cast $dst" %}
23059 ins_encode %{
23060 // empty
23061 %}
23062 ins_pipe(empty);
23063 %}
23064
23065 instruct vmaskcast_avx(vec dst, vec src) %{
23066 predicate(Matcher::vector_length_in_bytes(n) != Matcher::vector_length_in_bytes(n->in(1)));
23067 match(Set dst (VectorMaskCast src));
23068 format %{ "vector_mask_cast $dst, $src" %}
23069 ins_encode %{
23070 int vlen = Matcher::vector_length(this);
23071 BasicType src_bt = Matcher::vector_element_basic_type(this, $src);
23072 BasicType dst_bt = Matcher::vector_element_basic_type(this);
23073 __ vector_mask_cast($dst$$XMMRegister, $src$$XMMRegister, dst_bt, src_bt, vlen);
23074 %}
23075 ins_pipe(pipe_slow);
23076 %}
23077
23078 //-------------------------------- Load Iota Indices ----------------------------------
23079
23080 instruct loadIotaIndices(vec dst, immI_0 src) %{
23081 match(Set dst (VectorLoadConst src));
23082 format %{ "vector_load_iota $dst CONSTANT_MEMORY\t! load iota indices" %}
23083 ins_encode %{
23084 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
23085 BasicType bt = Matcher::vector_element_basic_type(this);
23086 __ load_iota_indices($dst$$XMMRegister, vlen_in_bytes, bt);
23087 %}
23088 ins_pipe( pipe_slow );
23089 %}
23090
23091 instruct VectorPopulateIndex(vec dst, rRegI src1, immI_1 src2, vec vtmp) %{
23092 match(Set dst (PopulateIndex src1 src2));
23093 effect(TEMP dst, TEMP vtmp);
23094 format %{ "vector_populate_index $dst $src1 $src2\t! using $vtmp as TEMP" %}
23095 ins_encode %{
23096 assert($src2$$constant == 1, "required");
23097 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
23098 int vlen_enc = vector_length_encoding(this);
23099 BasicType elem_bt = Matcher::vector_element_basic_type(this);
23100 __ vpbroadcast(elem_bt, $vtmp$$XMMRegister, $src1$$Register, vlen_enc);
23101 __ load_iota_indices($dst$$XMMRegister, vlen_in_bytes, elem_bt);
23102 __ vpadd(elem_bt, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
23103 %}
23104 ins_pipe( pipe_slow );
23105 %}
23106
23107 instruct VectorPopulateLIndex(vec dst, rRegL src1, immI_1 src2, vec vtmp) %{
23108 match(Set dst (PopulateIndex src1 src2));
23109 effect(TEMP dst, TEMP vtmp);
23110 format %{ "vector_populate_index $dst $src1 $src2\t! using $vtmp as TEMP" %}
23111 ins_encode %{
23112 assert($src2$$constant == 1, "required");
23113 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
23114 int vlen_enc = vector_length_encoding(this);
23115 BasicType elem_bt = Matcher::vector_element_basic_type(this);
23116 __ vpbroadcast(elem_bt, $vtmp$$XMMRegister, $src1$$Register, vlen_enc);
23117 __ load_iota_indices($dst$$XMMRegister, vlen_in_bytes, elem_bt);
23118 __ vpadd(elem_bt, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
23119 %}
23120 ins_pipe( pipe_slow );
23121 %}
23122
23123 //-------------------------------- Rearrange ----------------------------------
23124
23125 // LoadShuffle/Rearrange for Byte
23126 instruct rearrangeB(vec dst, vec shuffle) %{
23127 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
23128 Matcher::vector_length(n) < 32);
23129 match(Set dst (VectorRearrange dst shuffle));
23130 format %{ "vector_rearrange $dst, $shuffle, $dst" %}
23131 ins_encode %{
23132 assert(UseSSE >= 4, "required");
23133 __ pshufb($dst$$XMMRegister, $shuffle$$XMMRegister);
23134 %}
23135 ins_pipe( pipe_slow );
23136 %}
23137
23138 instruct rearrangeB_avx(legVec dst, legVec src, vec shuffle, legVec vtmp1, legVec vtmp2) %{
23139 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
23140 Matcher::vector_length(n) == 32 && !VM_Version::supports_avx512_vbmi());
23141 match(Set dst (VectorRearrange src shuffle));
23142 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
23143 format %{ "vector_rearrange $dst, $shuffle, $src\t! using $vtmp1, $vtmp2 as TEMP" %}
23144 ins_encode %{
23145 assert(UseAVX >= 2, "required");
23146 // Swap src into vtmp1
23147 __ vperm2i128($vtmp1$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 1);
23148 // Shuffle swapped src to get entries from other 128 bit lane
23149 __ vpshufb($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
23150 // Shuffle original src to get entries from self 128 bit lane
23151 __ vpshufb($dst$$XMMRegister, $src$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
23152 // Create a blend mask by setting high bits for entries coming from other lane in shuffle
23153 __ vpaddb($vtmp2$$XMMRegister, $shuffle$$XMMRegister, ExternalAddress(vector_byte_shufflemask()), Assembler::AVX_256bit, noreg);
23154 // Perform the blend
23155 __ vpblendvb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, Assembler::AVX_256bit);
23156 %}
23157 ins_pipe( pipe_slow );
23158 %}
23159
23160
23161 instruct rearrangeB_evex(vec dst, vec src, vec shuffle, vec xtmp1, vec xtmp2, vec xtmp3, kReg ktmp, rRegI rtmp) %{
23162 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
23163 Matcher::vector_length(n) > 32 && !VM_Version::supports_avx512_vbmi());
23164 match(Set dst (VectorRearrange src shuffle));
23165 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP ktmp, TEMP rtmp);
23166 format %{ "vector_rearrange $dst, $shuffle, $src!\t using $xtmp1, $xtmp2, $xtmp3, $rtmp and $ktmp as TEMP" %}
23167 ins_encode %{
23168 int vlen_enc = vector_length_encoding(this);
23169 __ rearrange_bytes($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister,
23170 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $xtmp3$$XMMRegister,
23171 $rtmp$$Register, $ktmp$$KRegister, vlen_enc);
23172 %}
23173 ins_pipe( pipe_slow );
23174 %}
23175
23176 instruct rearrangeB_evex_vbmi(vec dst, vec src, vec shuffle) %{
23177 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
23178 Matcher::vector_length(n) >= 32 && VM_Version::supports_avx512_vbmi());
23179 match(Set dst (VectorRearrange src shuffle));
23180 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23181 ins_encode %{
23182 int vlen_enc = vector_length_encoding(this);
23183 __ vpermb($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23184 %}
23185 ins_pipe( pipe_slow );
23186 %}
23187
23188 // LoadShuffle/Rearrange for Short
23189
23190 instruct loadShuffleS(vec dst, vec src, vec vtmp) %{
23191 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
23192 !VM_Version::supports_avx512bw());
23193 match(Set dst (VectorLoadShuffle src));
23194 effect(TEMP dst, TEMP vtmp);
23195 format %{ "vector_load_shuffle $dst, $src\t! using $vtmp as TEMP" %}
23196 ins_encode %{
23197 // Create a byte shuffle mask from short shuffle mask
23198 // only byte shuffle instruction available on these platforms
23199 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
23200 if (UseAVX == 0) {
23201 assert(vlen_in_bytes <= 16, "required");
23202 // Multiply each shuffle by two to get byte index
23203 __ movdqu($vtmp$$XMMRegister, $src$$XMMRegister);
23204 __ psllw($vtmp$$XMMRegister, 1);
23205
23206 // Duplicate to create 2 copies of byte index
23207 __ movdqu($dst$$XMMRegister, $vtmp$$XMMRegister);
23208 __ psllw($dst$$XMMRegister, 8);
23209 __ por($dst$$XMMRegister, $vtmp$$XMMRegister);
23210
23211 // Add one to get alternate byte index
23212 __ movdqu($vtmp$$XMMRegister, ExternalAddress(vector_short_shufflemask()), noreg);
23213 __ paddb($dst$$XMMRegister, $vtmp$$XMMRegister);
23214 } else {
23215 assert(UseAVX > 1 || vlen_in_bytes <= 16, "required");
23216 int vlen_enc = vector_length_encoding(this);
23217 // Multiply each shuffle by two to get byte index
23218 __ vpsllw($vtmp$$XMMRegister, $src$$XMMRegister, 1, vlen_enc);
23219
23220 // Duplicate to create 2 copies of byte index
23221 __ vpsllw($dst$$XMMRegister, $vtmp$$XMMRegister, 8, vlen_enc);
23222 __ vpor($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
23223
23224 // Add one to get alternate byte index
23225 __ vpaddb($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_short_shufflemask()), vlen_enc, noreg);
23226 }
23227 %}
23228 ins_pipe( pipe_slow );
23229 %}
23230
23231 instruct rearrangeS(vec dst, vec shuffle) %{
23232 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
23233 Matcher::vector_length(n) <= 8 && !VM_Version::supports_avx512bw());
23234 match(Set dst (VectorRearrange dst shuffle));
23235 format %{ "vector_rearrange $dst, $shuffle, $dst" %}
23236 ins_encode %{
23237 assert(UseSSE >= 4, "required");
23238 __ pshufb($dst$$XMMRegister, $shuffle$$XMMRegister);
23239 %}
23240 ins_pipe( pipe_slow );
23241 %}
23242
23243 instruct rearrangeS_avx(legVec dst, legVec src, vec shuffle, legVec vtmp1, legVec vtmp2) %{
23244 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
23245 Matcher::vector_length(n) == 16 && !VM_Version::supports_avx512bw());
23246 match(Set dst (VectorRearrange src shuffle));
23247 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
23248 format %{ "vector_rearrange $dst, $shuffle, $src\t! using $vtmp1, $vtmp2 as TEMP" %}
23249 ins_encode %{
23250 assert(UseAVX >= 2, "required");
23251 // Swap src into vtmp1
23252 __ vperm2i128($vtmp1$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 1);
23253 // Shuffle swapped src to get entries from other 128 bit lane
23254 __ vpshufb($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
23255 // Shuffle original src to get entries from self 128 bit lane
23256 __ vpshufb($dst$$XMMRegister, $src$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
23257 // Create a blend mask by setting high bits for entries coming from other lane in shuffle
23258 __ vpaddb($vtmp2$$XMMRegister, $shuffle$$XMMRegister, ExternalAddress(vector_byte_shufflemask()), Assembler::AVX_256bit, noreg);
23259 // Perform the blend
23260 __ vpblendvb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, Assembler::AVX_256bit);
23261 %}
23262 ins_pipe( pipe_slow );
23263 %}
23264
23265 instruct rearrangeS_evex(vec dst, vec src, vec shuffle) %{
23266 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
23267 VM_Version::supports_avx512bw());
23268 match(Set dst (VectorRearrange src shuffle));
23269 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23270 ins_encode %{
23271 int vlen_enc = vector_length_encoding(this);
23272 if (!VM_Version::supports_avx512vl()) {
23273 vlen_enc = Assembler::AVX_512bit;
23274 }
23275 __ vpermw($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23276 %}
23277 ins_pipe( pipe_slow );
23278 %}
23279
23280 // LoadShuffle/Rearrange for Integer and Float
23281
23282 instruct loadShuffleI(vec dst, vec src, vec vtmp) %{
23283 predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
23284 Matcher::vector_length(n) == 4 && UseAVX == 0);
23285 match(Set dst (VectorLoadShuffle src));
23286 effect(TEMP dst, TEMP vtmp);
23287 format %{ "vector_load_shuffle $dst, $src\t! using $vtmp as TEMP" %}
23288 ins_encode %{
23289 assert(UseSSE >= 4, "required");
23290
23291 // Create a byte shuffle mask from int shuffle mask
23292 // only byte shuffle instruction available on these platforms
23293
23294 // Duplicate and multiply each shuffle by 4
23295 __ movdqu($vtmp$$XMMRegister, $src$$XMMRegister);
23296 __ pshuflw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 0xA0);
23297 __ pshufhw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 0xA0);
23298 __ psllw($vtmp$$XMMRegister, 2);
23299
23300 // Duplicate again to create 4 copies of byte index
23301 __ movdqu($dst$$XMMRegister, $vtmp$$XMMRegister);
23302 __ psllw($dst$$XMMRegister, 8);
23303 __ por($vtmp$$XMMRegister, $dst$$XMMRegister);
23304
23305 // Add 3,2,1,0 to get alternate byte index
23306 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_int_shufflemask()), noreg);
23307 __ paddb($dst$$XMMRegister, $vtmp$$XMMRegister);
23308 %}
23309 ins_pipe( pipe_slow );
23310 %}
23311
23312 instruct rearrangeI(vec dst, vec shuffle) %{
23313 predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
23314 UseAVX == 0);
23315 match(Set dst (VectorRearrange dst shuffle));
23316 format %{ "vector_rearrange $dst, $shuffle, $dst" %}
23317 ins_encode %{
23318 assert(UseSSE >= 4, "required");
23319 __ pshufb($dst$$XMMRegister, $shuffle$$XMMRegister);
23320 %}
23321 ins_pipe( pipe_slow );
23322 %}
23323
23324 instruct rearrangeI_avx(vec dst, vec src, vec shuffle) %{
23325 predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
23326 UseAVX > 0);
23327 match(Set dst (VectorRearrange src shuffle));
23328 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23329 ins_encode %{
23330 int vlen_enc = vector_length_encoding(this);
23331 BasicType bt = Matcher::vector_element_basic_type(this);
23332 __ vector_rearrange_int_float(bt, $dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23333 %}
23334 ins_pipe( pipe_slow );
23335 %}
23336
23337 // LoadShuffle/Rearrange for Long and Double
23338
23339 instruct loadShuffleL(vec dst, vec src, vec vtmp) %{
23340 predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
23341 Matcher::vector_length(n) < 8 && !VM_Version::supports_avx512vl());
23342 match(Set dst (VectorLoadShuffle src));
23343 effect(TEMP dst, TEMP vtmp);
23344 format %{ "vector_load_shuffle $dst, $src\t! using $vtmp as TEMP" %}
23345 ins_encode %{
23346 assert(UseAVX >= 2, "required");
23347
23348 int vlen_enc = vector_length_encoding(this);
23349 // Create a double word shuffle mask from long shuffle mask
23350 // only double word shuffle instruction available on these platforms
23351
23352 // Multiply each shuffle by two to get double word index
23353 __ vpsllq($vtmp$$XMMRegister, $src$$XMMRegister, 1, vlen_enc);
23354
23355 // Duplicate each double word shuffle
23356 __ vpsllq($dst$$XMMRegister, $vtmp$$XMMRegister, 32, vlen_enc);
23357 __ vpor($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
23358
23359 // Add one to get alternate double word index
23360 __ vpaddd($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_long_shufflemask()), vlen_enc, noreg);
23361 %}
23362 ins_pipe( pipe_slow );
23363 %}
23364
23365 instruct rearrangeL(vec dst, vec src, vec shuffle) %{
23366 predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
23367 Matcher::vector_length(n) < 8 && !VM_Version::supports_avx512vl());
23368 match(Set dst (VectorRearrange src shuffle));
23369 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23370 ins_encode %{
23371 assert(UseAVX >= 2, "required");
23372
23373 int vlen_enc = vector_length_encoding(this);
23374 __ vpermd($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23375 %}
23376 ins_pipe( pipe_slow );
23377 %}
23378
23379 instruct rearrangeL_evex(vec dst, vec src, vec shuffle) %{
23380 predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
23381 (Matcher::vector_length(n) == 8 || VM_Version::supports_avx512vl()));
23382 match(Set dst (VectorRearrange src shuffle));
23383 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23384 ins_encode %{
23385 assert(UseAVX > 2, "required");
23386
23387 int vlen_enc = vector_length_encoding(this);
23388 if (vlen_enc == Assembler::AVX_128bit) {
23389 vlen_enc = Assembler::AVX_256bit;
23390 }
23391 __ vpermq($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23392 %}
23393 ins_pipe( pipe_slow );
23394 %}
23395
23396 // --------------------------------- FMA --------------------------------------
23397 // a * b + c
23398
23399 instruct vfmaF_reg(vec a, vec b, vec c) %{
23400 match(Set c (FmaVF c (Binary a b)));
23401 format %{ "fmaps $a,$b,$c\t# $c = $a * $b + $c fma packedF" %}
23402 ins_cost(150);
23403 ins_encode %{
23404 assert(UseFMA, "not enabled");
23405 int vlen_enc = vector_length_encoding(this);
23406 __ vfmaf($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister, vlen_enc);
23407 %}
23408 ins_pipe( pipe_slow );
23409 %}
23410
23411 instruct vfmaF_mem(vec a, memory b, vec c) %{
23412 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
23413 match(Set c (FmaVF c (Binary a (LoadVector b))));
23414 format %{ "fmaps $a,$b,$c\t# $c = $a * $b + $c fma packedF" %}
23415 ins_cost(150);
23416 ins_encode %{
23417 assert(UseFMA, "not enabled");
23418 int vlen_enc = vector_length_encoding(this);
23419 __ vfmaf($c$$XMMRegister, $a$$XMMRegister, $b$$Address, $c$$XMMRegister, vlen_enc);
23420 %}
23421 ins_pipe( pipe_slow );
23422 %}
23423
23424 instruct vfmaD_reg(vec a, vec b, vec c) %{
23425 match(Set c (FmaVD c (Binary a b)));
23426 format %{ "fmapd $a,$b,$c\t# $c = $a * $b + $c fma packedD" %}
23427 ins_cost(150);
23428 ins_encode %{
23429 assert(UseFMA, "not enabled");
23430 int vlen_enc = vector_length_encoding(this);
23431 __ vfmad($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister, vlen_enc);
23432 %}
23433 ins_pipe( pipe_slow );
23434 %}
23435
23436 instruct vfmaD_mem(vec a, memory b, vec c) %{
23437 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
23438 match(Set c (FmaVD c (Binary a (LoadVector b))));
23439 format %{ "fmapd $a,$b,$c\t# $c = $a * $b + $c fma packedD" %}
23440 ins_cost(150);
23441 ins_encode %{
23442 assert(UseFMA, "not enabled");
23443 int vlen_enc = vector_length_encoding(this);
23444 __ vfmad($c$$XMMRegister, $a$$XMMRegister, $b$$Address, $c$$XMMRegister, vlen_enc);
23445 %}
23446 ins_pipe( pipe_slow );
23447 %}
23448
23449 // --------------------------------- Vector Multiply Add --------------------------------------
23450
23451 instruct vmuladdS2I_reg_sse(vec dst, vec src1) %{
23452 predicate(UseAVX == 0);
23453 match(Set dst (MulAddVS2VI dst src1));
23454 format %{ "pmaddwd $dst,$src1\t! muladd packedStoI" %}
23455 ins_encode %{
23456 __ pmaddwd($dst$$XMMRegister, $src1$$XMMRegister);
23457 %}
23458 ins_pipe( pipe_slow );
23459 %}
23460
23461 instruct vmuladdS2I_reg_avx(vec dst, vec src1, vec src2) %{
23462 predicate(UseAVX > 0);
23463 match(Set dst (MulAddVS2VI src1 src2));
23464 format %{ "vpmaddwd $dst,$src1,$src2\t! muladd packedStoI" %}
23465 ins_encode %{
23466 int vlen_enc = vector_length_encoding(this);
23467 __ vpmaddwd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
23468 %}
23469 ins_pipe( pipe_slow );
23470 %}
23471
23472 // --------------------------------- Vector Multiply Add Add ----------------------------------
23473
23474 instruct vmuladdaddS2I_reg(vec dst, vec src1, vec src2) %{
23475 predicate(VM_Version::supports_avx512_vnni());
23476 match(Set dst (AddVI (MulAddVS2VI src1 src2) dst));
23477 format %{ "evpdpwssd $dst,$src1,$src2\t! muladdadd packedStoI" %}
23478 ins_encode %{
23479 assert(UseAVX > 2, "required");
23480 int vlen_enc = vector_length_encoding(this);
23481 __ evpdpwssd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
23482 %}
23483 ins_pipe( pipe_slow );
23484 ins_cost(10);
23485 %}
23486
23487 // --------------------------------- PopCount --------------------------------------
23488
23489 instruct vpopcount_integral_reg_evex(vec dst, vec src) %{
23490 predicate(is_vector_popcount_predicate(Matcher::vector_element_basic_type(n->in(1))));
23491 match(Set dst (PopCountVI src));
23492 match(Set dst (PopCountVL src));
23493 format %{ "vector_popcount_integral $dst, $src" %}
23494 ins_encode %{
23495 int opcode = this->ideal_Opcode();
23496 int vlen_enc = vector_length_encoding(this, $src);
23497 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23498 __ vector_popcount_integral_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, k0, true, vlen_enc);
23499 %}
23500 ins_pipe( pipe_slow );
23501 %}
23502
23503 instruct vpopcount_integral_reg_evex_masked(vec dst, vec src, kReg mask) %{
23504 predicate(is_vector_popcount_predicate(Matcher::vector_element_basic_type(n->in(1))));
23505 match(Set dst (PopCountVI src mask));
23506 match(Set dst (PopCountVL src mask));
23507 format %{ "vector_popcount_integral_masked $dst, $src, $mask" %}
23508 ins_encode %{
23509 int vlen_enc = vector_length_encoding(this, $src);
23510 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23511 __ evmovdquq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
23512 __ vector_popcount_integral_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $mask$$KRegister, true, vlen_enc);
23513 %}
23514 ins_pipe( pipe_slow );
23515 %}
23516
23517 instruct vpopcount_avx_reg(vec dst, vec src, vec xtmp1, vec xtmp2, rRegP rtmp) %{
23518 predicate(!is_vector_popcount_predicate(Matcher::vector_element_basic_type(n->in(1))));
23519 match(Set dst (PopCountVI src));
23520 match(Set dst (PopCountVL src));
23521 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp);
23522 format %{ "vector_popcount_integral $dst, $src\t! using $xtmp1, $xtmp2, and $rtmp as TEMP" %}
23523 ins_encode %{
23524 int opcode = this->ideal_Opcode();
23525 int vlen_enc = vector_length_encoding(this, $src);
23526 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23527 __ vector_popcount_integral(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23528 $xtmp2$$XMMRegister, $rtmp$$Register, vlen_enc);
23529 %}
23530 ins_pipe( pipe_slow );
23531 %}
23532
23533 // --------------------------------- Vector Trailing Zeros Count --------------------------------------
23534
23535 instruct vcount_trailing_zeros_reg_evex(vec dst, vec src, vec xtmp, rRegP rtmp) %{
23536 predicate(is_clz_non_subword_predicate_evex(Matcher::vector_element_basic_type(n->in(1)),
23537 Matcher::vector_length_in_bytes(n->in(1))));
23538 match(Set dst (CountTrailingZerosV src));
23539 effect(TEMP dst, TEMP xtmp, TEMP rtmp);
23540 ins_cost(400);
23541 format %{ "vector_count_trailing_zeros $dst, $src!\t using $xtmp and $rtmp as TEMP" %}
23542 ins_encode %{
23543 int vlen_enc = vector_length_encoding(this, $src);
23544 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23545 __ vector_count_trailing_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, xnoreg,
23546 xnoreg, xnoreg, $xtmp$$XMMRegister, k0, $rtmp$$Register, vlen_enc);
23547 %}
23548 ins_pipe( pipe_slow );
23549 %}
23550
23551 instruct vcount_trailing_zeros_short_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, rRegP rtmp) %{
23552 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_SHORT &&
23553 VM_Version::supports_avx512cd() &&
23554 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64));
23555 match(Set dst (CountTrailingZerosV src));
23556 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp);
23557 ins_cost(400);
23558 format %{ "vector_count_trailing_zeros $dst, $src!\t using $xtmp1, $xtmp2, $xtmp3 and $rtmp as TEMP" %}
23559 ins_encode %{
23560 int vlen_enc = vector_length_encoding(this, $src);
23561 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23562 __ vector_count_trailing_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23563 $xtmp2$$XMMRegister, xnoreg, $xtmp3$$XMMRegister, k0, $rtmp$$Register, vlen_enc);
23564 %}
23565 ins_pipe( pipe_slow );
23566 %}
23567
23568 instruct vcount_trailing_zeros_byte_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, kReg ktmp, rRegP rtmp) %{
23569 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_BYTE && VM_Version::supports_avx512vlbw());
23570 match(Set dst (CountTrailingZerosV src));
23571 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, TEMP ktmp, TEMP rtmp);
23572 ins_cost(400);
23573 format %{ "vector_count_trailing_zeros $dst, $src!\t using $xtmp1, $xtmp2, $xtmp3, $xtmp4, $ktmp and $rtmp as TEMP" %}
23574 ins_encode %{
23575 int vlen_enc = vector_length_encoding(this, $src);
23576 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23577 __ vector_count_trailing_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23578 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister,
23579 $ktmp$$KRegister, $rtmp$$Register, vlen_enc);
23580 %}
23581 ins_pipe( pipe_slow );
23582 %}
23583
23584 instruct vcount_trailing_zeros_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, rRegP rtmp) %{
23585 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n->in(1)) < 64);
23586 match(Set dst (CountTrailingZerosV src));
23587 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp);
23588 format %{ "vector_count_trailing_zeros $dst, $src\t! using $xtmp1, $xtmp2, $xtmp3, and $rtmp as TEMP" %}
23589 ins_encode %{
23590 int vlen_enc = vector_length_encoding(this, $src);
23591 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23592 __ vector_count_trailing_zeros_avx(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23593 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, vlen_enc);
23594 %}
23595 ins_pipe( pipe_slow );
23596 %}
23597
23598
23599 // --------------------------------- Bitwise Ternary Logic ----------------------------------
23600
23601 instruct vpternlog(vec dst, vec src2, vec src3, immU8 func) %{
23602 match(Set dst (MacroLogicV (Binary dst src2) (Binary src3 func)));
23603 effect(TEMP dst);
23604 format %{ "vpternlogd $dst,$src2,$src3,$func\t! vector ternary logic" %}
23605 ins_encode %{
23606 int vector_len = vector_length_encoding(this);
23607 __ vpternlogd($dst$$XMMRegister, $func$$constant, $src2$$XMMRegister, $src3$$XMMRegister, vector_len);
23608 %}
23609 ins_pipe( pipe_slow );
23610 %}
23611
23612 instruct vpternlog_mem(vec dst, vec src2, memory src3, immU8 func) %{
23613 predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) > 8);
23614 match(Set dst (MacroLogicV (Binary dst src2) (Binary (LoadVector src3) func)));
23615 effect(TEMP dst);
23616 format %{ "vpternlogd $dst,$src2,$src3,$func\t! vector ternary logic" %}
23617 ins_encode %{
23618 int vector_len = vector_length_encoding(this);
23619 __ vpternlogd($dst$$XMMRegister, $func$$constant, $src2$$XMMRegister, $src3$$Address, vector_len);
23620 %}
23621 ins_pipe( pipe_slow );
23622 %}
23623
23624 // --------------------------------- Rotation Operations ----------------------------------
23625 instruct vprotate_immI8(vec dst, vec src, immI8 shift) %{
23626 match(Set dst (RotateLeftV src shift));
23627 match(Set dst (RotateRightV src shift));
23628 format %{ "vprotate_imm8 $dst,$src,$shift\t! vector rotate" %}
23629 ins_encode %{
23630 int opcode = this->ideal_Opcode();
23631 int vector_len = vector_length_encoding(this);
23632 BasicType etype = this->bottom_type()->is_vect()->element_basic_type();
23633 __ vprotate_imm(opcode, etype, $dst$$XMMRegister, $src$$XMMRegister, $shift$$constant, vector_len);
23634 %}
23635 ins_pipe( pipe_slow );
23636 %}
23637
23638 instruct vprorate(vec dst, vec src, vec shift) %{
23639 match(Set dst (RotateLeftV src shift));
23640 match(Set dst (RotateRightV src shift));
23641 format %{ "vprotate $dst,$src,$shift\t! vector rotate" %}
23642 ins_encode %{
23643 int opcode = this->ideal_Opcode();
23644 int vector_len = vector_length_encoding(this);
23645 BasicType etype = this->bottom_type()->is_vect()->element_basic_type();
23646 __ vprotate_var(opcode, etype, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
23647 %}
23648 ins_pipe( pipe_slow );
23649 %}
23650
23651 // ---------------------------------- Masked Operations ------------------------------------
23652 instruct vmasked_load_avx_non_subword(vec dst, memory mem, vec mask) %{
23653 predicate(!n->in(3)->bottom_type()->isa_pvectmask());
23654 match(Set dst (LoadVectorMasked mem mask));
23655 format %{ "vector_masked_load $dst, $mem, $mask \t! vector masked copy" %}
23656 ins_encode %{
23657 BasicType elmType = this->bottom_type()->is_vect()->element_basic_type();
23658 int vlen_enc = vector_length_encoding(this);
23659 __ vmovmask(elmType, $dst$$XMMRegister, $mem$$Address, $mask$$XMMRegister, vlen_enc);
23660 %}
23661 ins_pipe( pipe_slow );
23662 %}
23663
23664
23665 instruct vmasked_load_evex(vec dst, memory mem, kReg mask) %{
23666 predicate(n->in(3)->bottom_type()->isa_pvectmask());
23667 match(Set dst (LoadVectorMasked mem mask));
23668 format %{ "vector_masked_load $dst, $mem, $mask \t! vector masked copy" %}
23669 ins_encode %{
23670 BasicType elmType = this->bottom_type()->is_vect()->element_basic_type();
23671 int vector_len = vector_length_encoding(this);
23672 __ evmovdqu(elmType, $mask$$KRegister, $dst$$XMMRegister, $mem$$Address, false, vector_len);
23673 %}
23674 ins_pipe( pipe_slow );
23675 %}
23676
23677 instruct vmasked_store_avx_non_subword(memory mem, vec src, vec mask) %{
23678 predicate(!n->in(3)->in(2)->bottom_type()->isa_pvectmask());
23679 match(Set mem (StoreVectorMasked mem (Binary src mask)));
23680 format %{ "vector_masked_store $mem, $src, $mask \t! vector masked store" %}
23681 ins_encode %{
23682 const MachNode* src_node = static_cast<const MachNode*>(this->in(this->operand_index($src)));
23683 int vlen_enc = vector_length_encoding(src_node);
23684 BasicType elmType = src_node->bottom_type()->is_vect()->element_basic_type();
23685 __ vmovmask(elmType, $mem$$Address, $src$$XMMRegister, $mask$$XMMRegister, vlen_enc);
23686 %}
23687 ins_pipe( pipe_slow );
23688 %}
23689
23690 instruct vmasked_store_evex(memory mem, vec src, kReg mask) %{
23691 predicate(n->in(3)->in(2)->bottom_type()->isa_pvectmask());
23692 match(Set mem (StoreVectorMasked mem (Binary src mask)));
23693 format %{ "vector_masked_store $mem, $src, $mask \t! vector masked store" %}
23694 ins_encode %{
23695 const MachNode* src_node = static_cast<const MachNode*>(this->in(this->operand_index($src)));
23696 BasicType elmType = src_node->bottom_type()->is_vect()->element_basic_type();
23697 int vlen_enc = vector_length_encoding(src_node);
23698 __ evmovdqu(elmType, $mask$$KRegister, $mem$$Address, $src$$XMMRegister, true, vlen_enc);
23699 %}
23700 ins_pipe( pipe_slow );
23701 %}
23702
23703 instruct verify_vector_alignment(rRegP addr, immL32 mask, rFlagsReg cr) %{
23704 match(Set addr (VerifyVectorAlignment addr mask));
23705 effect(KILL cr);
23706 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
23707 ins_encode %{
23708 Label Lskip;
23709 // check if masked bits of addr are zero
23710 __ testq($addr$$Register, $mask$$constant);
23711 __ jccb(Assembler::equal, Lskip);
23712 __ stop("verify_vector_alignment found a misaligned vector memory access");
23713 __ bind(Lskip);
23714 %}
23715 ins_pipe(pipe_slow);
23716 %}
23717
23718 instruct vmask_cmp_node(rRegI dst, vec src1, vec src2, kReg mask, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
23719 match(Set dst (VectorCmpMasked src1 (Binary src2 mask)));
23720 effect(TEMP_DEF dst, TEMP ktmp1, TEMP ktmp2, KILL cr);
23721 format %{ "vector_mask_cmp $src1, $src2, $mask \t! vector mask comparison" %}
23722 ins_encode %{
23723 assert(vector_length_encoding(this, $src1) == vector_length_encoding(this, $src2), "mismatch");
23724 assert(Matcher::vector_element_basic_type(this, $src1) == Matcher::vector_element_basic_type(this, $src2), "mismatch");
23725
23726 Label DONE;
23727 int vlen_enc = vector_length_encoding(this, $src1);
23728 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src1);
23729
23730 __ knotql($ktmp2$$KRegister, $mask$$KRegister);
23731 __ mov64($dst$$Register, -1L);
23732 __ evpcmp(elem_bt, $ktmp1$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, Assembler::eq, vlen_enc);
23733 __ kortestql($ktmp2$$KRegister, $ktmp1$$KRegister);
23734 __ jccb(Assembler::carrySet, DONE);
23735 __ kmovql($dst$$Register, $ktmp1$$KRegister);
23736 __ notq($dst$$Register);
23737 __ tzcntq($dst$$Register, $dst$$Register);
23738 __ bind(DONE);
23739 %}
23740 ins_pipe( pipe_slow );
23741 %}
23742
23743
23744 instruct vmask_gen(kReg dst, rRegL len, rRegL temp, rFlagsReg cr) %{
23745 match(Set dst (VectorMaskGen len));
23746 effect(TEMP temp, KILL cr);
23747 format %{ "vector_mask_gen32 $dst, $len \t! vector mask generator" %}
23748 ins_encode %{
23749 __ genmask($dst$$KRegister, $len$$Register, $temp$$Register);
23750 %}
23751 ins_pipe( pipe_slow );
23752 %}
23753
23754 instruct vmask_gen_imm(kReg dst, immL len, rRegL temp) %{
23755 match(Set dst (VectorMaskGen len));
23756 format %{ "vector_mask_gen $len \t! vector mask generator" %}
23757 effect(TEMP temp);
23758 ins_encode %{
23759 if ($len$$constant > 0) {
23760 __ mov64($temp$$Register, right_n_bits($len$$constant));
23761 __ kmovql($dst$$KRegister, $temp$$Register);
23762 } else {
23763 __ kxorql($dst$$KRegister, $dst$$KRegister, $dst$$KRegister);
23764 }
23765 %}
23766 ins_pipe( pipe_slow );
23767 %}
23768
23769 instruct vmask_tolong_evex(rRegL dst, kReg mask, rFlagsReg cr) %{
23770 predicate(n->in(1)->bottom_type()->isa_pvectmask());
23771 match(Set dst (VectorMaskToLong mask));
23772 effect(TEMP dst, KILL cr);
23773 format %{ "vector_tolong_evex $dst, $mask \t! vector mask tolong" %}
23774 ins_encode %{
23775 int opcode = this->ideal_Opcode();
23776 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23777 int mask_len = Matcher::vector_length(this, $mask);
23778 int mask_size = mask_len * type2aelembytes(mbt);
23779 int vlen_enc = vector_length_encoding(this, $mask);
23780 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
23781 $dst$$Register, mask_len, mask_size, vlen_enc);
23782 %}
23783 ins_pipe( pipe_slow );
23784 %}
23785
23786 instruct vmask_tolong_bool(rRegL dst, vec mask, vec xtmp, rFlagsReg cr) %{
23787 predicate(n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23788 match(Set dst (VectorMaskToLong mask));
23789 format %{ "vector_tolong_bool $dst, $mask \t! using $xtmp as TEMP" %}
23790 effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
23791 ins_encode %{
23792 int opcode = this->ideal_Opcode();
23793 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23794 int mask_len = Matcher::vector_length(this, $mask);
23795 int vlen_enc = vector_length_encoding(this, $mask);
23796 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23797 $dst$$Register, mask_len, mbt, vlen_enc);
23798 %}
23799 ins_pipe( pipe_slow );
23800 %}
23801
23802 instruct vmask_tolong_avx(rRegL dst, vec mask, immI size, vec xtmp, rFlagsReg cr) %{
23803 predicate(n->in(1)->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23804 match(Set dst (VectorMaskToLong (VectorStoreMask mask size)));
23805 format %{ "vector_tolong_avx $dst, $mask \t! using $xtmp as TEMP" %}
23806 effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
23807 ins_encode %{
23808 int opcode = this->ideal_Opcode();
23809 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23810 int mask_len = Matcher::vector_length(this, $mask);
23811 int vlen_enc = vector_length_encoding(this, $mask);
23812 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23813 $dst$$Register, mask_len, mbt, vlen_enc);
23814 %}
23815 ins_pipe( pipe_slow );
23816 %}
23817
23818 instruct vmask_truecount_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
23819 predicate(n->in(1)->bottom_type()->isa_pvectmask());
23820 match(Set dst (VectorMaskTrueCount mask));
23821 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
23822 format %{ "vector_truecount_evex $dst, $mask \t! using $tmp as TEMP" %}
23823 ins_encode %{
23824 int opcode = this->ideal_Opcode();
23825 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23826 int mask_len = Matcher::vector_length(this, $mask);
23827 int mask_size = mask_len * type2aelembytes(mbt);
23828 int vlen_enc = vector_length_encoding(this, $mask);
23829 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
23830 $tmp$$Register, mask_len, mask_size, vlen_enc);
23831 %}
23832 ins_pipe( pipe_slow );
23833 %}
23834
23835 instruct vmask_truecount_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23836 predicate(n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23837 match(Set dst (VectorMaskTrueCount mask));
23838 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23839 format %{ "vector_truecount_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23840 ins_encode %{
23841 int opcode = this->ideal_Opcode();
23842 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23843 int mask_len = Matcher::vector_length(this, $mask);
23844 int vlen_enc = vector_length_encoding(this, $mask);
23845 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23846 $tmp$$Register, mask_len, mbt, vlen_enc);
23847 %}
23848 ins_pipe( pipe_slow );
23849 %}
23850
23851 instruct vmask_truecount_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23852 predicate(n->in(1)->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23853 match(Set dst (VectorMaskTrueCount (VectorStoreMask mask size)));
23854 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23855 format %{ "vector_truecount_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23856 ins_encode %{
23857 int opcode = this->ideal_Opcode();
23858 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23859 int mask_len = Matcher::vector_length(this, $mask);
23860 int vlen_enc = vector_length_encoding(this, $mask);
23861 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23862 $tmp$$Register, mask_len, mbt, vlen_enc);
23863 %}
23864 ins_pipe( pipe_slow );
23865 %}
23866
23867 instruct vmask_first_or_last_true_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
23868 predicate(n->in(1)->bottom_type()->isa_pvectmask());
23869 match(Set dst (VectorMaskFirstTrue mask));
23870 match(Set dst (VectorMaskLastTrue mask));
23871 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
23872 format %{ "vector_mask_first_or_last_true_evex $dst, $mask \t! using $tmp as TEMP" %}
23873 ins_encode %{
23874 int opcode = this->ideal_Opcode();
23875 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23876 int mask_len = Matcher::vector_length(this, $mask);
23877 int mask_size = mask_len * type2aelembytes(mbt);
23878 int vlen_enc = vector_length_encoding(this, $mask);
23879 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
23880 $tmp$$Register, mask_len, mask_size, vlen_enc);
23881 %}
23882 ins_pipe( pipe_slow );
23883 %}
23884
23885 instruct vmask_first_or_last_true_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23886 predicate(n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23887 match(Set dst (VectorMaskFirstTrue mask));
23888 match(Set dst (VectorMaskLastTrue mask));
23889 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23890 format %{ "vector_mask_first_or_last_true_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23891 ins_encode %{
23892 int opcode = this->ideal_Opcode();
23893 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23894 int mask_len = Matcher::vector_length(this, $mask);
23895 int vlen_enc = vector_length_encoding(this, $mask);
23896 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23897 $tmp$$Register, mask_len, mbt, vlen_enc);
23898 %}
23899 ins_pipe( pipe_slow );
23900 %}
23901
23902 instruct vmask_first_or_last_true_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23903 predicate(n->in(1)->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23904 match(Set dst (VectorMaskFirstTrue (VectorStoreMask mask size)));
23905 match(Set dst (VectorMaskLastTrue (VectorStoreMask mask size)));
23906 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23907 format %{ "vector_mask_first_or_last_true_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23908 ins_encode %{
23909 int opcode = this->ideal_Opcode();
23910 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23911 int mask_len = Matcher::vector_length(this, $mask);
23912 int vlen_enc = vector_length_encoding(this, $mask);
23913 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23914 $tmp$$Register, mask_len, mbt, vlen_enc);
23915 %}
23916 ins_pipe( pipe_slow );
23917 %}
23918
23919 // --------------------------------- Compress/Expand Operations ---------------------------
23920 instruct vcompress_reg_avx(vec dst, vec src, vec mask, rRegI rtmp, rRegL rscratch, vec perm, vec xtmp, rFlagsReg cr) %{
23921 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n) <= 32);
23922 match(Set dst (CompressV src mask));
23923 match(Set dst (ExpandV src mask));
23924 effect(TEMP_DEF dst, TEMP perm, TEMP xtmp, TEMP rtmp, TEMP rscratch, KILL cr);
23925 format %{ "vector_compress $dst, $src, $mask \t!using $xtmp, $rtmp, $rscratch and $perm as TEMP" %}
23926 ins_encode %{
23927 int opcode = this->ideal_Opcode();
23928 int vlen_enc = vector_length_encoding(this);
23929 BasicType bt = Matcher::vector_element_basic_type(this);
23930 __ vector_compress_expand_avx2(opcode, $dst$$XMMRegister, $src$$XMMRegister, $mask$$XMMRegister, $rtmp$$Register,
23931 $rscratch$$Register, $perm$$XMMRegister, $xtmp$$XMMRegister, bt, vlen_enc);
23932 %}
23933 ins_pipe( pipe_slow );
23934 %}
23935
23936 instruct vcompress_expand_reg_evex(vec dst, vec src, kReg mask) %{
23937 predicate(VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64);
23938 match(Set dst (CompressV src mask));
23939 match(Set dst (ExpandV src mask));
23940 format %{ "vector_compress_expand $dst, $src, $mask" %}
23941 ins_encode %{
23942 int opcode = this->ideal_Opcode();
23943 int vector_len = vector_length_encoding(this);
23944 BasicType bt = Matcher::vector_element_basic_type(this);
23945 __ vector_compress_expand(opcode, $dst$$XMMRegister, $src$$XMMRegister, $mask$$KRegister, false, bt, vector_len);
23946 %}
23947 ins_pipe( pipe_slow );
23948 %}
23949
23950 instruct vcompress_mask_reg_evex(kReg dst, kReg mask, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
23951 match(Set dst (CompressM mask));
23952 effect(TEMP rtmp1, TEMP rtmp2, KILL cr);
23953 format %{ "mask_compress_evex $dst, $mask\t! using $rtmp1 and $rtmp2 as TEMP" %}
23954 ins_encode %{
23955 assert(this->in(1)->bottom_type()->isa_pvectmask(), "");
23956 int mask_len = Matcher::vector_length(this);
23957 __ vector_mask_compress($dst$$KRegister, $mask$$KRegister, $rtmp1$$Register, $rtmp2$$Register, mask_len);
23958 %}
23959 ins_pipe( pipe_slow );
23960 %}
23961
23962 // -------------------------------- Bit and Byte Reversal Vector Operations ------------------------
23963
23964 instruct vreverse_reg(vec dst, vec src, vec xtmp1, vec xtmp2, rRegI rtmp) %{
23965 predicate(!VM_Version::supports_gfni());
23966 match(Set dst (ReverseV src));
23967 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp);
23968 format %{ "vector_reverse_bit_evex $dst, $src!\t using $xtmp1, $xtmp2 and $rtmp as TEMP" %}
23969 ins_encode %{
23970 int vec_enc = vector_length_encoding(this);
23971 BasicType bt = Matcher::vector_element_basic_type(this);
23972 __ vector_reverse_bit(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23973 $xtmp2$$XMMRegister, $rtmp$$Register, vec_enc);
23974 %}
23975 ins_pipe( pipe_slow );
23976 %}
23977
23978 instruct vreverse_reg_gfni(vec dst, vec src, vec xtmp) %{
23979 predicate(VM_Version::supports_gfni());
23980 match(Set dst (ReverseV src));
23981 effect(TEMP dst, TEMP xtmp);
23982 format %{ "vector_reverse_bit_gfni $dst, $src!\t using $xtmp as TEMP" %}
23983 ins_encode %{
23984 int vec_enc = vector_length_encoding(this);
23985 BasicType bt = Matcher::vector_element_basic_type(this);
23986 InternalAddress addr = $constantaddress(jlong(0x8040201008040201));
23987 __ vector_reverse_bit_gfni(bt, $dst$$XMMRegister, $src$$XMMRegister, addr, vec_enc,
23988 $xtmp$$XMMRegister);
23989 %}
23990 ins_pipe( pipe_slow );
23991 %}
23992
23993 instruct vreverse_byte_reg(vec dst, vec src) %{
23994 predicate(VM_Version::supports_avx512bw() || Matcher::vector_length_in_bytes(n) < 64);
23995 match(Set dst (ReverseBytesV src));
23996 effect(TEMP dst);
23997 format %{ "vector_reverse_byte $dst, $src" %}
23998 ins_encode %{
23999 int vec_enc = vector_length_encoding(this);
24000 BasicType bt = Matcher::vector_element_basic_type(this);
24001 __ vector_reverse_byte(bt, $dst$$XMMRegister, $src$$XMMRegister, vec_enc);
24002 %}
24003 ins_pipe( pipe_slow );
24004 %}
24005
24006 instruct vreverse_byte64_reg(vec dst, vec src, vec xtmp1, vec xtmp2, rRegI rtmp) %{
24007 predicate(!VM_Version::supports_avx512bw() && Matcher::vector_length_in_bytes(n) == 64);
24008 match(Set dst (ReverseBytesV src));
24009 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp);
24010 format %{ "vector_reverse_byte $dst, $src!\t using $xtmp1, $xtmp2 and $rtmp as TEMP" %}
24011 ins_encode %{
24012 int vec_enc = vector_length_encoding(this);
24013 BasicType bt = Matcher::vector_element_basic_type(this);
24014 __ vector_reverse_byte64(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
24015 $xtmp2$$XMMRegister, $rtmp$$Register, vec_enc);
24016 %}
24017 ins_pipe( pipe_slow );
24018 %}
24019
24020 // ---------------------------------- Vector Count Leading Zeros -----------------------------------
24021
24022 instruct vcount_leading_zeros_IL_reg_evex(vec dst, vec src) %{
24023 predicate(is_clz_non_subword_predicate_evex(Matcher::vector_element_basic_type(n->in(1)),
24024 Matcher::vector_length_in_bytes(n->in(1))));
24025 match(Set dst (CountLeadingZerosV src));
24026 format %{ "vector_count_leading_zeros $dst, $src" %}
24027 ins_encode %{
24028 int vlen_enc = vector_length_encoding(this, $src);
24029 BasicType bt = Matcher::vector_element_basic_type(this, $src);
24030 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, xnoreg,
24031 xnoreg, xnoreg, k0, noreg, true, vlen_enc);
24032 %}
24033 ins_pipe( pipe_slow );
24034 %}
24035
24036 instruct vcount_leading_zeros_IL_reg_evex_masked(vec dst, vec src, kReg mask) %{
24037 predicate(is_clz_non_subword_predicate_evex(Matcher::vector_element_basic_type(n->in(1)),
24038 Matcher::vector_length_in_bytes(n->in(1))));
24039 match(Set dst (CountLeadingZerosV src mask));
24040 format %{ "vector_count_leading_zeros $dst, $src, $mask" %}
24041 ins_encode %{
24042 int vlen_enc = vector_length_encoding(this, $src);
24043 BasicType bt = Matcher::vector_element_basic_type(this, $src);
24044 __ evmovdquq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
24045 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, xnoreg, xnoreg,
24046 xnoreg, $mask$$KRegister, noreg, true, vlen_enc);
24047 %}
24048 ins_pipe( pipe_slow );
24049 %}
24050
24051 instruct vcount_leading_zeros_short_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2) %{
24052 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_SHORT &&
24053 VM_Version::supports_avx512cd() &&
24054 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64));
24055 match(Set dst (CountLeadingZerosV src));
24056 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
24057 format %{ "vector_count_leading_zeros $dst, $src!\t using $xtmp1 and $xtmp2 as TEMP" %}
24058 ins_encode %{
24059 int vlen_enc = vector_length_encoding(this, $src);
24060 BasicType bt = Matcher::vector_element_basic_type(this, $src);
24061 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
24062 $xtmp2$$XMMRegister, xnoreg, k0, noreg, true, vlen_enc);
24063 %}
24064 ins_pipe( pipe_slow );
24065 %}
24066
24067 instruct vcount_leading_zeros_byte_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, kReg ktmp, rRegP rtmp) %{
24068 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_BYTE && VM_Version::supports_avx512vlbw());
24069 match(Set dst (CountLeadingZerosV src));
24070 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP ktmp, TEMP rtmp);
24071 format %{ "vector_count_leading_zeros $dst, $src!\t using $xtmp1, $xtmp2, $xtmp3, $ktmp and $rtmp as TEMP" %}
24072 ins_encode %{
24073 int vlen_enc = vector_length_encoding(this, $src);
24074 BasicType bt = Matcher::vector_element_basic_type(this, $src);
24075 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
24076 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $ktmp$$KRegister,
24077 $rtmp$$Register, true, vlen_enc);
24078 %}
24079 ins_pipe( pipe_slow );
24080 %}
24081
24082 instruct vcount_leading_zeros_int_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3) %{
24083 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_INT &&
24084 !VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n->in(1)) < 64);
24085 match(Set dst (CountLeadingZerosV src));
24086 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3);
24087 format %{ "vector_count_leading_zeros $dst, $src\t! using $xtmp1, $xtmp2 and $xtmp3 as TEMP" %}
24088 ins_encode %{
24089 int vlen_enc = vector_length_encoding(this, $src);
24090 BasicType bt = Matcher::vector_element_basic_type(this, $src);
24091 __ vector_count_leading_zeros_avx(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
24092 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, noreg, vlen_enc);
24093 %}
24094 ins_pipe( pipe_slow );
24095 %}
24096
24097 instruct vcount_leading_zeros_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, rRegP rtmp) %{
24098 predicate(Matcher::vector_element_basic_type(n->in(1)) != T_INT &&
24099 !VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n->in(1)) < 64);
24100 match(Set dst (CountLeadingZerosV src));
24101 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp);
24102 format %{ "vector_count_leading_zeros $dst, $src\t! using $xtmp1, $xtmp2, $xtmp3, and $rtmp as TEMP" %}
24103 ins_encode %{
24104 int vlen_enc = vector_length_encoding(this, $src);
24105 BasicType bt = Matcher::vector_element_basic_type(this, $src);
24106 __ vector_count_leading_zeros_avx(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
24107 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, vlen_enc);
24108 %}
24109 ins_pipe( pipe_slow );
24110 %}
24111
24112 // ---------------------------------- Vector Masked Operations ------------------------------------
24113
24114 instruct vadd_reg_masked(vec dst, vec src2, kReg mask) %{
24115 match(Set dst (AddVB (Binary dst src2) mask));
24116 match(Set dst (AddVS (Binary dst src2) mask));
24117 match(Set dst (AddVI (Binary dst src2) mask));
24118 match(Set dst (AddVL (Binary dst src2) mask));
24119 match(Set dst (AddVF (Binary dst src2) mask));
24120 match(Set dst (AddVD (Binary dst src2) mask));
24121 format %{ "vpadd_masked $dst, $dst, $src2, $mask\t! add masked operation" %}
24122 ins_encode %{
24123 int vlen_enc = vector_length_encoding(this);
24124 BasicType bt = Matcher::vector_element_basic_type(this);
24125 int opc = this->ideal_Opcode();
24126 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24127 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24128 %}
24129 ins_pipe( pipe_slow );
24130 %}
24131
24132 instruct vadd_mem_masked(vec dst, memory src2, kReg mask) %{
24133 match(Set dst (AddVB (Binary dst (LoadVector src2)) mask));
24134 match(Set dst (AddVS (Binary dst (LoadVector src2)) mask));
24135 match(Set dst (AddVI (Binary dst (LoadVector src2)) mask));
24136 match(Set dst (AddVL (Binary dst (LoadVector src2)) mask));
24137 match(Set dst (AddVF (Binary dst (LoadVector src2)) mask));
24138 match(Set dst (AddVD (Binary dst (LoadVector src2)) mask));
24139 format %{ "vpadd_masked $dst, $dst, $src2, $mask\t! add masked operation" %}
24140 ins_encode %{
24141 int vlen_enc = vector_length_encoding(this);
24142 BasicType bt = Matcher::vector_element_basic_type(this);
24143 int opc = this->ideal_Opcode();
24144 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24145 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24146 %}
24147 ins_pipe( pipe_slow );
24148 %}
24149
24150 instruct vxor_reg_masked(vec dst, vec src2, kReg mask) %{
24151 match(Set dst (XorV (Binary dst src2) mask));
24152 format %{ "vxor_masked $dst, $dst, $src2, $mask\t! xor masked operation" %}
24153 ins_encode %{
24154 int vlen_enc = vector_length_encoding(this);
24155 BasicType bt = Matcher::vector_element_basic_type(this);
24156 int opc = this->ideal_Opcode();
24157 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24158 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24159 %}
24160 ins_pipe( pipe_slow );
24161 %}
24162
24163 instruct vxor_mem_masked(vec dst, memory src2, kReg mask) %{
24164 match(Set dst (XorV (Binary dst (LoadVector src2)) mask));
24165 format %{ "vxor_masked $dst, $dst, $src2, $mask\t! xor masked operation" %}
24166 ins_encode %{
24167 int vlen_enc = vector_length_encoding(this);
24168 BasicType bt = Matcher::vector_element_basic_type(this);
24169 int opc = this->ideal_Opcode();
24170 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24171 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24172 %}
24173 ins_pipe( pipe_slow );
24174 %}
24175
24176 instruct vor_reg_masked(vec dst, vec src2, kReg mask) %{
24177 match(Set dst (OrV (Binary dst src2) mask));
24178 format %{ "vor_masked $dst, $dst, $src2, $mask\t! or masked operation" %}
24179 ins_encode %{
24180 int vlen_enc = vector_length_encoding(this);
24181 BasicType bt = Matcher::vector_element_basic_type(this);
24182 int opc = this->ideal_Opcode();
24183 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24184 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24185 %}
24186 ins_pipe( pipe_slow );
24187 %}
24188
24189 instruct vor_mem_masked(vec dst, memory src2, kReg mask) %{
24190 match(Set dst (OrV (Binary dst (LoadVector src2)) mask));
24191 format %{ "vor_masked $dst, $dst, $src2, $mask\t! or masked operation" %}
24192 ins_encode %{
24193 int vlen_enc = vector_length_encoding(this);
24194 BasicType bt = Matcher::vector_element_basic_type(this);
24195 int opc = this->ideal_Opcode();
24196 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24197 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24198 %}
24199 ins_pipe( pipe_slow );
24200 %}
24201
24202 instruct vand_reg_masked(vec dst, vec src2, kReg mask) %{
24203 match(Set dst (AndV (Binary dst src2) mask));
24204 format %{ "vand_masked $dst, $dst, $src2, $mask\t! and masked operation" %}
24205 ins_encode %{
24206 int vlen_enc = vector_length_encoding(this);
24207 BasicType bt = Matcher::vector_element_basic_type(this);
24208 int opc = this->ideal_Opcode();
24209 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24210 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24211 %}
24212 ins_pipe( pipe_slow );
24213 %}
24214
24215 instruct vand_mem_masked(vec dst, memory src2, kReg mask) %{
24216 match(Set dst (AndV (Binary dst (LoadVector src2)) mask));
24217 format %{ "vand_masked $dst, $dst, $src2, $mask\t! and masked operation" %}
24218 ins_encode %{
24219 int vlen_enc = vector_length_encoding(this);
24220 BasicType bt = Matcher::vector_element_basic_type(this);
24221 int opc = this->ideal_Opcode();
24222 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24223 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24224 %}
24225 ins_pipe( pipe_slow );
24226 %}
24227
24228 instruct vsub_reg_masked(vec dst, vec src2, kReg mask) %{
24229 match(Set dst (SubVB (Binary dst src2) mask));
24230 match(Set dst (SubVS (Binary dst src2) mask));
24231 match(Set dst (SubVI (Binary dst src2) mask));
24232 match(Set dst (SubVL (Binary dst src2) mask));
24233 match(Set dst (SubVF (Binary dst src2) mask));
24234 match(Set dst (SubVD (Binary dst src2) mask));
24235 format %{ "vpsub_masked $dst, $dst, $src2, $mask\t! sub masked operation" %}
24236 ins_encode %{
24237 int vlen_enc = vector_length_encoding(this);
24238 BasicType bt = Matcher::vector_element_basic_type(this);
24239 int opc = this->ideal_Opcode();
24240 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24241 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24242 %}
24243 ins_pipe( pipe_slow );
24244 %}
24245
24246 instruct vsub_mem_masked(vec dst, memory src2, kReg mask) %{
24247 match(Set dst (SubVB (Binary dst (LoadVector src2)) mask));
24248 match(Set dst (SubVS (Binary dst (LoadVector src2)) mask));
24249 match(Set dst (SubVI (Binary dst (LoadVector src2)) mask));
24250 match(Set dst (SubVL (Binary dst (LoadVector src2)) mask));
24251 match(Set dst (SubVF (Binary dst (LoadVector src2)) mask));
24252 match(Set dst (SubVD (Binary dst (LoadVector src2)) mask));
24253 format %{ "vpsub_masked $dst, $dst, $src2, $mask\t! sub masked operation" %}
24254 ins_encode %{
24255 int vlen_enc = vector_length_encoding(this);
24256 BasicType bt = Matcher::vector_element_basic_type(this);
24257 int opc = this->ideal_Opcode();
24258 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24259 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24260 %}
24261 ins_pipe( pipe_slow );
24262 %}
24263
24264 instruct vmul_reg_masked(vec dst, vec src2, kReg mask) %{
24265 match(Set dst (MulVS (Binary dst src2) mask));
24266 match(Set dst (MulVI (Binary dst src2) mask));
24267 match(Set dst (MulVL (Binary dst src2) mask));
24268 match(Set dst (MulVF (Binary dst src2) mask));
24269 match(Set dst (MulVD (Binary dst src2) mask));
24270 format %{ "vpmul_masked $dst, $dst, $src2, $mask\t! mul masked operation" %}
24271 ins_encode %{
24272 int vlen_enc = vector_length_encoding(this);
24273 BasicType bt = Matcher::vector_element_basic_type(this);
24274 int opc = this->ideal_Opcode();
24275 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24276 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24277 %}
24278 ins_pipe( pipe_slow );
24279 %}
24280
24281 instruct vmul_mem_masked(vec dst, memory src2, kReg mask) %{
24282 match(Set dst (MulVS (Binary dst (LoadVector src2)) mask));
24283 match(Set dst (MulVI (Binary dst (LoadVector src2)) mask));
24284 match(Set dst (MulVL (Binary dst (LoadVector src2)) mask));
24285 match(Set dst (MulVF (Binary dst (LoadVector src2)) mask));
24286 match(Set dst (MulVD (Binary dst (LoadVector src2)) mask));
24287 format %{ "vpmul_masked $dst, $dst, $src2, $mask\t! mul masked operation" %}
24288 ins_encode %{
24289 int vlen_enc = vector_length_encoding(this);
24290 BasicType bt = Matcher::vector_element_basic_type(this);
24291 int opc = this->ideal_Opcode();
24292 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24293 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24294 %}
24295 ins_pipe( pipe_slow );
24296 %}
24297
24298 instruct vsqrt_reg_masked(vec dst, kReg mask) %{
24299 match(Set dst (SqrtVF dst mask));
24300 match(Set dst (SqrtVD dst mask));
24301 format %{ "vpsqrt_masked $dst, $mask\t! sqrt masked operation" %}
24302 ins_encode %{
24303 int vlen_enc = vector_length_encoding(this);
24304 BasicType bt = Matcher::vector_element_basic_type(this);
24305 int opc = this->ideal_Opcode();
24306 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24307 $dst$$XMMRegister, $dst$$XMMRegister, true, vlen_enc);
24308 %}
24309 ins_pipe( pipe_slow );
24310 %}
24311
24312 instruct vdiv_reg_masked(vec dst, vec src2, kReg mask) %{
24313 match(Set dst (DivVF (Binary dst src2) mask));
24314 match(Set dst (DivVD (Binary dst src2) mask));
24315 format %{ "vpdiv_masked $dst, $dst, $src2, $mask\t! div masked operation" %}
24316 ins_encode %{
24317 int vlen_enc = vector_length_encoding(this);
24318 BasicType bt = Matcher::vector_element_basic_type(this);
24319 int opc = this->ideal_Opcode();
24320 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24321 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24322 %}
24323 ins_pipe( pipe_slow );
24324 %}
24325
24326 instruct vdiv_mem_masked(vec dst, memory src2, kReg mask) %{
24327 match(Set dst (DivVF (Binary dst (LoadVector src2)) mask));
24328 match(Set dst (DivVD (Binary dst (LoadVector src2)) mask));
24329 format %{ "vpdiv_masked $dst, $dst, $src2, $mask\t! div masked operation" %}
24330 ins_encode %{
24331 int vlen_enc = vector_length_encoding(this);
24332 BasicType bt = Matcher::vector_element_basic_type(this);
24333 int opc = this->ideal_Opcode();
24334 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24335 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24336 %}
24337 ins_pipe( pipe_slow );
24338 %}
24339
24340
24341 instruct vrol_imm_masked(vec dst, immI8 shift, kReg mask) %{
24342 match(Set dst (RotateLeftV (Binary dst shift) mask));
24343 match(Set dst (RotateRightV (Binary dst shift) mask));
24344 format %{ "vprotate_imm_masked $dst, $dst, $shift, $mask\t! rotate masked operation" %}
24345 ins_encode %{
24346 int vlen_enc = vector_length_encoding(this);
24347 BasicType bt = Matcher::vector_element_basic_type(this);
24348 int opc = this->ideal_Opcode();
24349 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24350 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24351 %}
24352 ins_pipe( pipe_slow );
24353 %}
24354
24355 instruct vrol_reg_masked(vec dst, vec src2, kReg mask) %{
24356 match(Set dst (RotateLeftV (Binary dst src2) mask));
24357 match(Set dst (RotateRightV (Binary dst src2) mask));
24358 format %{ "vrotate_masked $dst, $dst, $src2, $mask\t! rotate masked operation" %}
24359 ins_encode %{
24360 int vlen_enc = vector_length_encoding(this);
24361 BasicType bt = Matcher::vector_element_basic_type(this);
24362 int opc = this->ideal_Opcode();
24363 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24364 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24365 %}
24366 ins_pipe( pipe_slow );
24367 %}
24368
24369 instruct vlshift_imm_masked(vec dst, immI8 shift, kReg mask) %{
24370 match(Set dst (LShiftVS (Binary dst (LShiftCntV shift)) mask));
24371 match(Set dst (LShiftVI (Binary dst (LShiftCntV shift)) mask));
24372 match(Set dst (LShiftVL (Binary dst (LShiftCntV shift)) mask));
24373 format %{ "vplshift_imm_masked $dst, $dst, $shift, $mask\t! lshift masked operation" %}
24374 ins_encode %{
24375 int vlen_enc = vector_length_encoding(this);
24376 BasicType bt = Matcher::vector_element_basic_type(this);
24377 int opc = this->ideal_Opcode();
24378 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24379 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24380 %}
24381 ins_pipe( pipe_slow );
24382 %}
24383
24384 instruct vlshift_reg_masked(vec dst, vec src2, kReg mask) %{
24385 predicate(!n->as_ShiftV()->is_var_shift());
24386 match(Set dst (LShiftVS (Binary dst src2) mask));
24387 match(Set dst (LShiftVI (Binary dst src2) mask));
24388 match(Set dst (LShiftVL (Binary dst src2) mask));
24389 format %{ "vplshift_masked $dst, $dst, $src2, $mask\t! lshift masked operation" %}
24390 ins_encode %{
24391 int vlen_enc = vector_length_encoding(this);
24392 BasicType bt = Matcher::vector_element_basic_type(this);
24393 int opc = this->ideal_Opcode();
24394 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24395 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, false);
24396 %}
24397 ins_pipe( pipe_slow );
24398 %}
24399
24400 instruct vlshiftv_reg_masked(vec dst, vec src2, kReg mask) %{
24401 predicate(n->as_ShiftV()->is_var_shift());
24402 match(Set dst (LShiftVS (Binary dst src2) mask));
24403 match(Set dst (LShiftVI (Binary dst src2) mask));
24404 match(Set dst (LShiftVL (Binary dst src2) mask));
24405 format %{ "vplshiftv_masked $dst, $dst, $src2, $mask\t! lshift masked operation" %}
24406 ins_encode %{
24407 int vlen_enc = vector_length_encoding(this);
24408 BasicType bt = Matcher::vector_element_basic_type(this);
24409 int opc = this->ideal_Opcode();
24410 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24411 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, true);
24412 %}
24413 ins_pipe( pipe_slow );
24414 %}
24415
24416 instruct vrshift_imm_masked(vec dst, immI8 shift, kReg mask) %{
24417 match(Set dst (RShiftVS (Binary dst (RShiftCntV shift)) mask));
24418 match(Set dst (RShiftVI (Binary dst (RShiftCntV shift)) mask));
24419 match(Set dst (RShiftVL (Binary dst (RShiftCntV shift)) mask));
24420 format %{ "vprshift_imm_masked $dst, $dst, $shift, $mask\t! rshift masked operation" %}
24421 ins_encode %{
24422 int vlen_enc = vector_length_encoding(this);
24423 BasicType bt = Matcher::vector_element_basic_type(this);
24424 int opc = this->ideal_Opcode();
24425 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24426 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24427 %}
24428 ins_pipe( pipe_slow );
24429 %}
24430
24431 instruct vrshift_reg_masked(vec dst, vec src2, kReg mask) %{
24432 predicate(!n->as_ShiftV()->is_var_shift());
24433 match(Set dst (RShiftVS (Binary dst src2) mask));
24434 match(Set dst (RShiftVI (Binary dst src2) mask));
24435 match(Set dst (RShiftVL (Binary dst src2) mask));
24436 format %{ "vprshift_masked $dst, $dst, $src2, $mask\t! rshift masked operation" %}
24437 ins_encode %{
24438 int vlen_enc = vector_length_encoding(this);
24439 BasicType bt = Matcher::vector_element_basic_type(this);
24440 int opc = this->ideal_Opcode();
24441 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24442 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, false);
24443 %}
24444 ins_pipe( pipe_slow );
24445 %}
24446
24447 instruct vrshiftv_reg_masked(vec dst, vec src2, kReg mask) %{
24448 predicate(n->as_ShiftV()->is_var_shift());
24449 match(Set dst (RShiftVS (Binary dst src2) mask));
24450 match(Set dst (RShiftVI (Binary dst src2) mask));
24451 match(Set dst (RShiftVL (Binary dst src2) mask));
24452 format %{ "vprshiftv_masked $dst, $dst, $src2, $mask\t! rshift masked operation" %}
24453 ins_encode %{
24454 int vlen_enc = vector_length_encoding(this);
24455 BasicType bt = Matcher::vector_element_basic_type(this);
24456 int opc = this->ideal_Opcode();
24457 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24458 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, true);
24459 %}
24460 ins_pipe( pipe_slow );
24461 %}
24462
24463 instruct vurshift_imm_masked(vec dst, immI8 shift, kReg mask) %{
24464 match(Set dst (URShiftVS (Binary dst (RShiftCntV shift)) mask));
24465 match(Set dst (URShiftVI (Binary dst (RShiftCntV shift)) mask));
24466 match(Set dst (URShiftVL (Binary dst (RShiftCntV shift)) mask));
24467 format %{ "vpurshift_imm_masked $dst, $dst, $shift, $mask\t! urshift masked operation" %}
24468 ins_encode %{
24469 int vlen_enc = vector_length_encoding(this);
24470 BasicType bt = Matcher::vector_element_basic_type(this);
24471 int opc = this->ideal_Opcode();
24472 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24473 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24474 %}
24475 ins_pipe( pipe_slow );
24476 %}
24477
24478 instruct vurshift_reg_masked(vec dst, vec src2, kReg mask) %{
24479 predicate(!n->as_ShiftV()->is_var_shift());
24480 match(Set dst (URShiftVS (Binary dst src2) mask));
24481 match(Set dst (URShiftVI (Binary dst src2) mask));
24482 match(Set dst (URShiftVL (Binary dst src2) mask));
24483 format %{ "vpurshift_masked $dst, $dst, $src2, $mask\t! urshift masked operation" %}
24484 ins_encode %{
24485 int vlen_enc = vector_length_encoding(this);
24486 BasicType bt = Matcher::vector_element_basic_type(this);
24487 int opc = this->ideal_Opcode();
24488 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24489 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, false);
24490 %}
24491 ins_pipe( pipe_slow );
24492 %}
24493
24494 instruct vurshiftv_reg_masked(vec dst, vec src2, kReg mask) %{
24495 predicate(n->as_ShiftV()->is_var_shift());
24496 match(Set dst (URShiftVS (Binary dst src2) mask));
24497 match(Set dst (URShiftVI (Binary dst src2) mask));
24498 match(Set dst (URShiftVL (Binary dst src2) mask));
24499 format %{ "vpurshiftv_masked $dst, $dst, $src2, $mask\t! urshift masked operation" %}
24500 ins_encode %{
24501 int vlen_enc = vector_length_encoding(this);
24502 BasicType bt = Matcher::vector_element_basic_type(this);
24503 int opc = this->ideal_Opcode();
24504 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24505 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, true);
24506 %}
24507 ins_pipe( pipe_slow );
24508 %}
24509
24510 instruct vmaxv_reg_masked(vec dst, vec src2, kReg mask) %{
24511 match(Set dst (MaxV (Binary dst src2) mask));
24512 format %{ "vpmax_masked $dst, $dst, $src2, $mask\t! max masked operation" %}
24513 ins_encode %{
24514 int vlen_enc = vector_length_encoding(this);
24515 BasicType bt = Matcher::vector_element_basic_type(this);
24516 int opc = this->ideal_Opcode();
24517 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24518 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24519 %}
24520 ins_pipe( pipe_slow );
24521 %}
24522
24523 instruct vmaxv_mem_masked(vec dst, memory src2, kReg mask) %{
24524 match(Set dst (MaxV (Binary dst (LoadVector src2)) mask));
24525 format %{ "vpmax_masked $dst, $dst, $src2, $mask\t! max masked operation" %}
24526 ins_encode %{
24527 int vlen_enc = vector_length_encoding(this);
24528 BasicType bt = Matcher::vector_element_basic_type(this);
24529 int opc = this->ideal_Opcode();
24530 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24531 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24532 %}
24533 ins_pipe( pipe_slow );
24534 %}
24535
24536 instruct vminv_reg_masked(vec dst, vec src2, kReg mask) %{
24537 match(Set dst (MinV (Binary dst src2) mask));
24538 format %{ "vpmin_masked $dst, $dst, $src2, $mask\t! min masked operation" %}
24539 ins_encode %{
24540 int vlen_enc = vector_length_encoding(this);
24541 BasicType bt = Matcher::vector_element_basic_type(this);
24542 int opc = this->ideal_Opcode();
24543 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24544 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24545 %}
24546 ins_pipe( pipe_slow );
24547 %}
24548
24549 instruct vminv_mem_masked(vec dst, memory src2, kReg mask) %{
24550 match(Set dst (MinV (Binary dst (LoadVector src2)) mask));
24551 format %{ "vpmin_masked $dst, $dst, $src2, $mask\t! min masked operation" %}
24552 ins_encode %{
24553 int vlen_enc = vector_length_encoding(this);
24554 BasicType bt = Matcher::vector_element_basic_type(this);
24555 int opc = this->ideal_Opcode();
24556 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24557 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24558 %}
24559 ins_pipe( pipe_slow );
24560 %}
24561
24562 instruct vrearrangev_reg_masked(vec dst, vec src2, kReg mask) %{
24563 match(Set dst (VectorRearrange (Binary dst src2) mask));
24564 format %{ "vprearrange_masked $dst, $dst, $src2, $mask\t! rearrange masked operation" %}
24565 ins_encode %{
24566 int vlen_enc = vector_length_encoding(this);
24567 BasicType bt = Matcher::vector_element_basic_type(this);
24568 int opc = this->ideal_Opcode();
24569 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24570 $dst$$XMMRegister, $src2$$XMMRegister, false, vlen_enc);
24571 %}
24572 ins_pipe( pipe_slow );
24573 %}
24574
24575 instruct vabs_masked(vec dst, kReg mask) %{
24576 match(Set dst (AbsVB dst mask));
24577 match(Set dst (AbsVS dst mask));
24578 match(Set dst (AbsVI dst mask));
24579 match(Set dst (AbsVL dst mask));
24580 format %{ "vabs_masked $dst, $mask \t! vabs masked operation" %}
24581 ins_encode %{
24582 int vlen_enc = vector_length_encoding(this);
24583 BasicType bt = Matcher::vector_element_basic_type(this);
24584 int opc = this->ideal_Opcode();
24585 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24586 $dst$$XMMRegister, $dst$$XMMRegister, true, vlen_enc);
24587 %}
24588 ins_pipe( pipe_slow );
24589 %}
24590
24591 instruct vfma_reg_masked(vec dst, vec src2, vec src3, kReg mask) %{
24592 match(Set dst (FmaVF (Binary dst src2) (Binary src3 mask)));
24593 match(Set dst (FmaVD (Binary dst src2) (Binary src3 mask)));
24594 format %{ "vfma_masked $dst, $src2, $src3, $mask \t! vfma masked operation" %}
24595 ins_encode %{
24596 assert(UseFMA, "Needs FMA instructions support.");
24597 int vlen_enc = vector_length_encoding(this);
24598 BasicType bt = Matcher::vector_element_basic_type(this);
24599 int opc = this->ideal_Opcode();
24600 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24601 $src2$$XMMRegister, $src3$$XMMRegister, true, vlen_enc);
24602 %}
24603 ins_pipe( pipe_slow );
24604 %}
24605
24606 instruct vfma_mem_masked(vec dst, vec src2, memory src3, kReg mask) %{
24607 match(Set dst (FmaVF (Binary dst src2) (Binary (LoadVector src3) mask)));
24608 match(Set dst (FmaVD (Binary dst src2) (Binary (LoadVector src3) mask)));
24609 format %{ "vfma_masked $dst, $src2, $src3, $mask \t! vfma masked operation" %}
24610 ins_encode %{
24611 assert(UseFMA, "Needs FMA instructions support.");
24612 int vlen_enc = vector_length_encoding(this);
24613 BasicType bt = Matcher::vector_element_basic_type(this);
24614 int opc = this->ideal_Opcode();
24615 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24616 $src2$$XMMRegister, $src3$$Address, true, vlen_enc);
24617 %}
24618 ins_pipe( pipe_slow );
24619 %}
24620
24621 instruct evcmp_masked(kReg dst, vec src1, vec src2, immI8 cond, kReg mask) %{
24622 match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond mask)));
24623 format %{ "vcmp_masked $dst, $src1, $src2, $cond, $mask" %}
24624 ins_encode %{
24625 assert(bottom_type()->isa_pvectmask(), "TypePVectMask expected");
24626 int vlen_enc = vector_length_encoding(this, $src1);
24627 BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
24628
24629 // Comparison i
24630 switch (src1_elem_bt) {
24631 case T_BYTE: {
24632 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24633 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24634 __ evpcmpb($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24635 break;
24636 }
24637 case T_SHORT: {
24638 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24639 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24640 __ evpcmpw($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24641 break;
24642 }
24643 case T_INT: {
24644 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24645 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24646 __ evpcmpd($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24647 break;
24648 }
24649 case T_LONG: {
24650 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24651 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24652 __ evpcmpq($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24653 break;
24654 }
24655 case T_FLOAT: {
24656 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
24657 __ evcmpps($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
24658 break;
24659 }
24660 case T_DOUBLE: {
24661 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
24662 __ evcmppd($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
24663 break;
24664 }
24665 default: assert(false, "%s", type2name(src1_elem_bt)); break;
24666 }
24667 %}
24668 ins_pipe( pipe_slow );
24669 %}
24670
24671 instruct mask_all_evexI_LE32(kReg dst, rRegI src) %{
24672 predicate(Matcher::vector_length(n) <= 32);
24673 match(Set dst (MaskAll src));
24674 format %{ "mask_all_evexI_LE32 $dst, $src \t" %}
24675 ins_encode %{
24676 int mask_len = Matcher::vector_length(this);
24677 __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
24678 %}
24679 ins_pipe( pipe_slow );
24680 %}
24681
24682 instruct mask_not_immLT8(kReg dst, kReg src, rRegI rtmp, kReg ktmp, immI_M1 cnt) %{
24683 predicate(Matcher::vector_length(n) < 8 && VM_Version::supports_avx512dq());
24684 match(Set dst (XorVMask src (MaskAll cnt)));
24685 effect(TEMP_DEF dst, TEMP rtmp, TEMP ktmp);
24686 format %{ "mask_not_LT8 $dst, $src, $cnt \t!using $ktmp and $rtmp as TEMP" %}
24687 ins_encode %{
24688 uint masklen = Matcher::vector_length(this);
24689 __ knot(masklen, $dst$$KRegister, $src$$KRegister, $ktmp$$KRegister, $rtmp$$Register);
24690 %}
24691 ins_pipe( pipe_slow );
24692 %}
24693
24694 instruct mask_not_imm(kReg dst, kReg src, immI_M1 cnt) %{
24695 predicate((Matcher::vector_length(n) == 8 && VM_Version::supports_avx512dq()) ||
24696 (Matcher::vector_length(n) == 16) ||
24697 (Matcher::vector_length(n) > 16 && VM_Version::supports_avx512bw()));
24698 match(Set dst (XorVMask src (MaskAll cnt)));
24699 format %{ "mask_not $dst, $src, $cnt \t! mask not operation" %}
24700 ins_encode %{
24701 uint masklen = Matcher::vector_length(this);
24702 __ knot(masklen, $dst$$KRegister, $src$$KRegister);
24703 %}
24704 ins_pipe( pipe_slow );
24705 %}
24706
24707 instruct long_to_maskLE8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2) %{
24708 predicate(n->bottom_type()->isa_pvectmask() == nullptr && Matcher::vector_length(n) <= 8);
24709 match(Set dst (VectorLongToMask src));
24710 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2);
24711 format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2" %}
24712 ins_encode %{
24713 int mask_len = Matcher::vector_length(this);
24714 int vec_enc = vector_length_encoding(mask_len);
24715 __ vector_long_to_maskvec($dst$$XMMRegister, $src$$Register, $rtmp1$$Register,
24716 $rtmp2$$Register, xnoreg, mask_len, vec_enc);
24717 %}
24718 ins_pipe( pipe_slow );
24719 %}
24720
24721
24722 instruct long_to_maskGT8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec xtmp1, rFlagsReg cr) %{
24723 predicate(n->bottom_type()->isa_pvectmask() == nullptr && Matcher::vector_length(n) > 8);
24724 match(Set dst (VectorLongToMask src));
24725 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, TEMP xtmp1, KILL cr);
24726 format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2, $xtmp1, as TEMP" %}
24727 ins_encode %{
24728 int mask_len = Matcher::vector_length(this);
24729 assert(mask_len <= 32, "invalid mask length");
24730 int vec_enc = vector_length_encoding(mask_len);
24731 __ vector_long_to_maskvec($dst$$XMMRegister, $src$$Register, $rtmp1$$Register,
24732 $rtmp2$$Register, $xtmp1$$XMMRegister, mask_len, vec_enc);
24733 %}
24734 ins_pipe( pipe_slow );
24735 %}
24736
24737 instruct long_to_mask_evex(kReg dst, rRegL src) %{
24738 predicate(n->bottom_type()->isa_pvectmask());
24739 match(Set dst (VectorLongToMask src));
24740 format %{ "long_to_mask_evex $dst, $src\t!" %}
24741 ins_encode %{
24742 __ kmov($dst$$KRegister, $src$$Register);
24743 %}
24744 ins_pipe( pipe_slow );
24745 %}
24746
24747 instruct mask_opers_evex(kReg dst, kReg src1, kReg src2, kReg kscratch) %{
24748 match(Set dst (AndVMask src1 src2));
24749 match(Set dst (OrVMask src1 src2));
24750 match(Set dst (XorVMask src1 src2));
24751 effect(TEMP kscratch);
24752 format %{ "mask_opers_evex $dst, $src1, $src2\t! using $kscratch as TEMP" %}
24753 ins_encode %{
24754 const MachNode* mask1 = static_cast<const MachNode*>(this->in(this->operand_index($src1)));
24755 const MachNode* mask2 = static_cast<const MachNode*>(this->in(this->operand_index($src2)));
24756 assert(Type::equals(mask1->bottom_type(), mask2->bottom_type()), "Mask types must be equal");
24757 uint masklen = Matcher::vector_length(this);
24758 masklen = (masklen < 16 && !VM_Version::supports_avx512dq()) ? 16 : masklen;
24759 __ masked_op(this->ideal_Opcode(), masklen, $dst$$KRegister, $src1$$KRegister, $src2$$KRegister);
24760 %}
24761 ins_pipe( pipe_slow );
24762 %}
24763
24764 instruct vternlog_reg_masked(vec dst, vec src2, vec src3, immU8 func, kReg mask) %{
24765 match(Set dst (MacroLogicV dst (Binary src2 (Binary src3 (Binary func mask)))));
24766 format %{ "vternlog_masked $dst,$src2,$src3,$func,$mask\t! vternlog masked operation" %}
24767 ins_encode %{
24768 int vlen_enc = vector_length_encoding(this);
24769 BasicType bt = Matcher::vector_element_basic_type(this);
24770 __ evpternlog($dst$$XMMRegister, $func$$constant, $mask$$KRegister,
24771 $src2$$XMMRegister, $src3$$XMMRegister, true, bt, vlen_enc);
24772 %}
24773 ins_pipe( pipe_slow );
24774 %}
24775
24776 instruct vternlogd_mem_masked(vec dst, vec src2, memory src3, immU8 func, kReg mask) %{
24777 match(Set dst (MacroLogicV dst (Binary src2 (Binary src3 (Binary func mask)))));
24778 format %{ "vternlog_masked $dst,$src2,$src3,$func,$mask\t! vternlog masked operation" %}
24779 ins_encode %{
24780 int vlen_enc = vector_length_encoding(this);
24781 BasicType bt = Matcher::vector_element_basic_type(this);
24782 __ evpternlog($dst$$XMMRegister, $func$$constant, $mask$$KRegister,
24783 $src2$$XMMRegister, $src3$$Address, true, bt, vlen_enc);
24784 %}
24785 ins_pipe( pipe_slow );
24786 %}
24787
24788 instruct castMM(kReg dst)
24789 %{
24790 match(Set dst (CastVV dst));
24791
24792 size(0);
24793 format %{ "# castVV of $dst" %}
24794 ins_encode(/* empty encoding */);
24795 ins_cost(0);
24796 ins_pipe(empty);
24797 %}
24798
24799 instruct castVV(vec dst)
24800 %{
24801 match(Set dst (CastVV dst));
24802
24803 size(0);
24804 format %{ "# castVV of $dst" %}
24805 ins_encode(/* empty encoding */);
24806 ins_cost(0);
24807 ins_pipe(empty);
24808 %}
24809
24810 instruct castVVLeg(legVec dst)
24811 %{
24812 match(Set dst (CastVV dst));
24813
24814 size(0);
24815 format %{ "# castVV of $dst" %}
24816 ins_encode(/* empty encoding */);
24817 ins_cost(0);
24818 ins_pipe(empty);
24819 %}
24820
24821 instruct FloatClassCheck_reg_reg_vfpclass(rRegI dst, regF src, kReg ktmp, rFlagsReg cr)
24822 %{
24823 match(Set dst (IsInfiniteF src));
24824 effect(TEMP ktmp, KILL cr);
24825 format %{ "float_class_check $dst, $src" %}
24826 ins_encode %{
24827 __ vfpclassss($ktmp$$KRegister, $src$$XMMRegister, 0x18);
24828 __ kmovbl($dst$$Register, $ktmp$$KRegister);
24829 %}
24830 ins_pipe(pipe_slow);
24831 %}
24832
24833 instruct DoubleClassCheck_reg_reg_vfpclass(rRegI dst, regD src, kReg ktmp, rFlagsReg cr)
24834 %{
24835 match(Set dst (IsInfiniteD src));
24836 effect(TEMP ktmp, KILL cr);
24837 format %{ "double_class_check $dst, $src" %}
24838 ins_encode %{
24839 __ vfpclasssd($ktmp$$KRegister, $src$$XMMRegister, 0x18);
24840 __ kmovbl($dst$$Register, $ktmp$$KRegister);
24841 %}
24842 ins_pipe(pipe_slow);
24843 %}
24844
24845 instruct vector_addsub_saturating_subword_reg(vec dst, vec src1, vec src2)
24846 %{
24847 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24848 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
24849 match(Set dst (SaturatingAddV src1 src2));
24850 match(Set dst (SaturatingSubV src1 src2));
24851 format %{ "vector_addsub_saturating_subword $dst, $src1, $src2" %}
24852 ins_encode %{
24853 int vlen_enc = vector_length_encoding(this);
24854 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24855 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24856 $src1$$XMMRegister, $src2$$XMMRegister, false, vlen_enc);
24857 %}
24858 ins_pipe(pipe_slow);
24859 %}
24860
24861 instruct vector_addsub_saturating_unsigned_subword_reg(vec dst, vec src1, vec src2)
24862 %{
24863 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24864 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
24865 match(Set dst (SaturatingAddV src1 src2));
24866 match(Set dst (SaturatingSubV src1 src2));
24867 format %{ "vector_addsub_saturating_unsigned_subword $dst, $src1, $src2" %}
24868 ins_encode %{
24869 int vlen_enc = vector_length_encoding(this);
24870 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24871 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24872 $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24873 %}
24874 ins_pipe(pipe_slow);
24875 %}
24876
24877 instruct vector_addsub_saturating_reg_evex(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2)
24878 %{
24879 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24880 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned() &&
24881 (Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()));
24882 match(Set dst (SaturatingAddV src1 src2));
24883 match(Set dst (SaturatingSubV src1 src2));
24884 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2);
24885 format %{ "vector_addsub_saturating_evex $dst, $src1, $src2 \t! using $xtmp1, $xtmp2, $ktmp1 and $ktmp2 as TEMP" %}
24886 ins_encode %{
24887 int vlen_enc = vector_length_encoding(this);
24888 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24889 __ vector_addsub_dq_saturating_evex(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24890 $src1$$XMMRegister, $src2$$XMMRegister,
24891 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister,
24892 $ktmp1$$KRegister, $ktmp2$$KRegister, vlen_enc);
24893 %}
24894 ins_pipe(pipe_slow);
24895 %}
24896
24897 instruct vector_addsub_saturating_reg_avx(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4)
24898 %{
24899 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24900 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned() &&
24901 Matcher::vector_length_in_bytes(n) <= 32 && !VM_Version::supports_avx512vl());
24902 match(Set dst (SaturatingAddV src1 src2));
24903 match(Set dst (SaturatingSubV src1 src2));
24904 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4);
24905 format %{ "vector_addsub_saturating_avx $dst, $src1, $src2 \t! using $xtmp1, $xtmp2, $xtmp3 and $xtmp4 as TEMP" %}
24906 ins_encode %{
24907 int vlen_enc = vector_length_encoding(this);
24908 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24909 __ vector_addsub_dq_saturating_avx(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister, $src1$$XMMRegister,
24910 $src2$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister,
24911 $xtmp3$$XMMRegister, $xtmp4$$XMMRegister, vlen_enc);
24912 %}
24913 ins_pipe(pipe_slow);
24914 %}
24915
24916 instruct vector_add_saturating_unsigned_reg_evex(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, kReg ktmp)
24917 %{
24918 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24919 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24920 (Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()));
24921 match(Set dst (SaturatingAddV src1 src2));
24922 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp);
24923 format %{ "vector_add_saturating_unsigned_evex $dst, $src1, $src2 \t! using $xtmp1, $xtmp2 and $ktmp as TEMP" %}
24924 ins_encode %{
24925 int vlen_enc = vector_length_encoding(this);
24926 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24927 __ vector_add_dq_saturating_unsigned_evex(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
24928 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp$$KRegister, vlen_enc);
24929 %}
24930 ins_pipe(pipe_slow);
24931 %}
24932
24933 instruct vector_add_saturating_unsigned_reg_avx(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, vec xtmp3)
24934 %{
24935 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24936 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24937 Matcher::vector_length_in_bytes(n) <= 32 && !VM_Version::supports_avx512vl());
24938 match(Set dst (SaturatingAddV src1 src2));
24939 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3);
24940 format %{ "vector_add_saturating_unsigned_avx $dst, $src1, $src2 \t! using $xtmp1, $xtmp2 and $xtmp3 as TEMP" %}
24941 ins_encode %{
24942 int vlen_enc = vector_length_encoding(this);
24943 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24944 __ vector_add_dq_saturating_unsigned_avx(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
24945 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, vlen_enc);
24946 %}
24947 ins_pipe(pipe_slow);
24948 %}
24949
24950 instruct vector_sub_saturating_unsigned_reg_evex(vec dst, vec src1, vec src2, kReg ktmp)
24951 %{
24952 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24953 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24954 (Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()));
24955 match(Set dst (SaturatingSubV src1 src2));
24956 effect(TEMP ktmp);
24957 format %{ "vector_sub_saturating_unsigned_evex $dst, $src1, $src2 \t! using $ktmp as TEMP" %}
24958 ins_encode %{
24959 int vlen_enc = vector_length_encoding(this);
24960 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24961 __ vector_sub_dq_saturating_unsigned_evex(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister,
24962 $src2$$XMMRegister, $ktmp$$KRegister, vlen_enc);
24963 %}
24964 ins_pipe(pipe_slow);
24965 %}
24966
24967 instruct vector_sub_saturating_unsigned_reg_avx(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2)
24968 %{
24969 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24970 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24971 Matcher::vector_length_in_bytes(n) <= 32 && !VM_Version::supports_avx512vl());
24972 match(Set dst (SaturatingSubV src1 src2));
24973 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
24974 format %{ "vector_sub_saturating_unsigned_avx $dst, $src1, $src2 \t! using $xtmp1 and $xtmp2 as TEMP" %}
24975 ins_encode %{
24976 int vlen_enc = vector_length_encoding(this);
24977 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24978 __ vector_sub_dq_saturating_unsigned_avx(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
24979 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
24980 %}
24981 ins_pipe(pipe_slow);
24982 %}
24983
24984 instruct vector_addsub_saturating_subword_mem(vec dst, vec src1, memory src2)
24985 %{
24986 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24987 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
24988 match(Set dst (SaturatingAddV src1 (LoadVector src2)));
24989 match(Set dst (SaturatingSubV src1 (LoadVector src2)));
24990 format %{ "vector_addsub_saturating_subword $dst, $src1, $src2" %}
24991 ins_encode %{
24992 int vlen_enc = vector_length_encoding(this);
24993 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24994 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24995 $src1$$XMMRegister, $src2$$Address, false, vlen_enc);
24996 %}
24997 ins_pipe(pipe_slow);
24998 %}
24999
25000 instruct vector_addsub_saturating_unsigned_subword_mem(vec dst, vec src1, memory src2)
25001 %{
25002 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
25003 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
25004 match(Set dst (SaturatingAddV src1 (LoadVector src2)));
25005 match(Set dst (SaturatingSubV src1 (LoadVector src2)));
25006 format %{ "vector_addsub_saturating_unsigned_subword $dst, $src1, $src2" %}
25007 ins_encode %{
25008 int vlen_enc = vector_length_encoding(this);
25009 BasicType elem_bt = Matcher::vector_element_basic_type(this);
25010 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
25011 $src1$$XMMRegister, $src2$$Address, true, vlen_enc);
25012 %}
25013 ins_pipe(pipe_slow);
25014 %}
25015
25016 instruct vector_addsub_saturating_subword_masked_reg(vec dst, vec src, kReg mask) %{
25017 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
25018 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
25019 match(Set dst (SaturatingAddV (Binary dst src) mask));
25020 match(Set dst (SaturatingSubV (Binary dst src) mask));
25021 format %{ "vector_addsub_saturating_subword_masked $dst, $mask, $src" %}
25022 ins_encode %{
25023 int vlen_enc = vector_length_encoding(this);
25024 BasicType elem_bt = Matcher::vector_element_basic_type(this);
25025 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
25026 $dst$$XMMRegister, $src$$XMMRegister, false, true, vlen_enc);
25027 %}
25028 ins_pipe( pipe_slow );
25029 %}
25030
25031 instruct vector_addsub_saturating_unsigned_subword_masked_reg(vec dst, vec src, kReg mask) %{
25032 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
25033 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
25034 match(Set dst (SaturatingAddV (Binary dst src) mask));
25035 match(Set dst (SaturatingSubV (Binary dst src) mask));
25036 format %{ "vector_addsub_saturating_unsigned_subword_masked $dst, $mask, $src" %}
25037 ins_encode %{
25038 int vlen_enc = vector_length_encoding(this);
25039 BasicType elem_bt = Matcher::vector_element_basic_type(this);
25040 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
25041 $dst$$XMMRegister, $src$$XMMRegister, true, true, vlen_enc);
25042 %}
25043 ins_pipe( pipe_slow );
25044 %}
25045
25046 instruct vector_addsub_saturating_subword_masked_mem(vec dst, memory src, kReg mask) %{
25047 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
25048 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
25049 match(Set dst (SaturatingAddV (Binary dst (LoadVector src)) mask));
25050 match(Set dst (SaturatingSubV (Binary dst (LoadVector src)) mask));
25051 format %{ "vector_addsub_saturating_subword_masked $dst, $mask, $src" %}
25052 ins_encode %{
25053 int vlen_enc = vector_length_encoding(this);
25054 BasicType elem_bt = Matcher::vector_element_basic_type(this);
25055 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
25056 $dst$$XMMRegister, $src$$Address, false, true, vlen_enc);
25057 %}
25058 ins_pipe( pipe_slow );
25059 %}
25060
25061 instruct vector_addsub_saturating_unsigned_subword_masked_mem(vec dst, memory src, kReg mask) %{
25062 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
25063 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
25064 match(Set dst (SaturatingAddV (Binary dst (LoadVector src)) mask));
25065 match(Set dst (SaturatingSubV (Binary dst (LoadVector src)) mask));
25066 format %{ "vector_addsub_saturating_unsigned_subword_masked $dst, $mask, $src" %}
25067 ins_encode %{
25068 int vlen_enc = vector_length_encoding(this);
25069 BasicType elem_bt = Matcher::vector_element_basic_type(this);
25070 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
25071 $dst$$XMMRegister, $src$$Address, true, true, vlen_enc);
25072 %}
25073 ins_pipe( pipe_slow );
25074 %}
25075
25076 instruct vector_selectfrom_twovectors_reg_evex(vec index, vec src1, vec src2)
25077 %{
25078 match(Set index (SelectFromTwoVector (Binary index src1) src2));
25079 format %{ "select_from_two_vector $index, $src1, $src2 \t!" %}
25080 ins_encode %{
25081 int vlen_enc = vector_length_encoding(this);
25082 BasicType bt = Matcher::vector_element_basic_type(this);
25083 __ select_from_two_vectors_evex(bt, $index$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
25084 %}
25085 ins_pipe(pipe_slow);
25086 %}
25087
25088 instruct reinterpretS2HF(regF dst, rRegI src)
25089 %{
25090 match(Set dst (ReinterpretS2HF src));
25091 format %{ "evmovw $dst, $src" %}
25092 ins_encode %{
25093 __ evmovw($dst$$XMMRegister, $src$$Register);
25094 %}
25095 ins_pipe(pipe_slow);
25096 %}
25097
25098 instruct reinterpretHF2S(rRegI dst, regF src)
25099 %{
25100 match(Set dst (ReinterpretHF2S src));
25101 format %{ "evmovw $dst, $src" %}
25102 ins_encode %{
25103 __ evmovw($dst$$Register, $src$$XMMRegister);
25104 __ narrow_subword_type($dst$$Register, T_SHORT);
25105 %}
25106 ins_pipe(pipe_slow);
25107 %}
25108
25109 instruct convF2HFAndS2HF(regF dst, regF src)
25110 %{
25111 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
25112 format %{ "convF2HFAndS2HF $dst, $src" %}
25113 ins_encode %{
25114 __ vcvtps2ph($dst$$XMMRegister, $src$$XMMRegister, 0x04, Assembler::AVX_128bit);
25115 %}
25116 ins_pipe(pipe_slow);
25117 %}
25118
25119 instruct convHF2SAndHF2F(regF dst, regF src)
25120 %{
25121 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
25122 format %{ "convHF2SAndHF2F $dst, $src" %}
25123 ins_encode %{
25124 __ vcvtph2ps($dst$$XMMRegister, $src$$XMMRegister, Assembler::AVX_128bit);
25125 %}
25126 ins_pipe(pipe_slow);
25127 %}
25128
25129 instruct scalar_sqrt_HF_reg(regF dst, regF src)
25130 %{
25131 match(Set dst (SqrtHF src));
25132 format %{ "scalar_sqrt_fp16 $dst, $src" %}
25133 ins_encode %{
25134 __ vsqrtsh($dst$$XMMRegister, $src$$XMMRegister);
25135 %}
25136 ins_pipe(pipe_slow);
25137 %}
25138
25139 instruct scalar_binOps_HF_reg(regF dst, regF src1, regF src2)
25140 %{
25141 match(Set dst (AddHF src1 src2));
25142 match(Set dst (DivHF src1 src2));
25143 match(Set dst (MulHF src1 src2));
25144 match(Set dst (SubHF src1 src2));
25145 format %{ "scalar_binop_fp16 $dst, $src1, $src2" %}
25146 ins_encode %{
25147 int opcode = this->ideal_Opcode();
25148 __ efp16sh(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
25149 %}
25150 ins_pipe(pipe_slow);
25151 %}
25152
25153 instruct scalar_minmax_HF_reg_avx10_2(regF dst, regF src1, regF src2)
25154 %{
25155 predicate(VM_Version::supports_avx10_2());
25156 match(Set dst (MaxHF src1 src2));
25157 match(Set dst (MinHF src1 src2));
25158
25159 format %{ "scalar_min_max_fp16 $dst, $src1, $src2" %}
25160 ins_encode %{
25161 int opcode = this->ideal_Opcode();
25162 __ sminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, k0);
25163 %}
25164 ins_pipe( pipe_slow );
25165 %}
25166
25167 instruct scalar_minmax_HF_reg(regF dst, regF src1, regF src2, kReg ktmp, regF xtmp1, regF xtmp2)
25168 %{
25169 predicate(!VM_Version::supports_avx10_2());
25170 match(Set dst (MaxHF src1 src2));
25171 match(Set dst (MinHF src1 src2));
25172 effect(TEMP_DEF dst, TEMP ktmp, TEMP xtmp1, TEMP xtmp2);
25173
25174 format %{ "scalar_min_max_fp16 $dst, $src1, $src2\t using $ktmp, $xtmp1 and $xtmp2 as TEMP" %}
25175 ins_encode %{
25176 int opcode = this->ideal_Opcode();
25177 __ sminmax_fp16(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $ktmp$$KRegister,
25178 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
25179 %}
25180 ins_pipe( pipe_slow );
25181 %}
25182
25183 instruct scalar_fma_HF_reg(regF dst, regF src1, regF src2)
25184 %{
25185 match(Set dst (FmaHF src2 (Binary dst src1)));
25186 effect(DEF dst);
25187 format %{ "scalar_fma_fp16 $dst, $src1, $src2\t# $dst = $dst * $src1 + $src2 fma packedH" %}
25188 ins_encode %{
25189 __ vfmadd132sh($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister);
25190 %}
25191 ins_pipe( pipe_slow );
25192 %}
25193
25194
25195 instruct vector_sqrt_HF_reg(vec dst, vec src)
25196 %{
25197 match(Set dst (SqrtVHF src));
25198 format %{ "vector_sqrt_fp16 $dst, $src" %}
25199 ins_encode %{
25200 int vlen_enc = vector_length_encoding(this);
25201 __ evsqrtph($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
25202 %}
25203 ins_pipe(pipe_slow);
25204 %}
25205
25206 instruct vector_sqrt_HF_mem(vec dst, memory src)
25207 %{
25208 match(Set dst (SqrtVHF (VectorReinterpret (LoadVector src))));
25209 format %{ "vector_sqrt_fp16_mem $dst, $src" %}
25210 ins_encode %{
25211 int vlen_enc = vector_length_encoding(this);
25212 __ evsqrtph($dst$$XMMRegister, $src$$Address, vlen_enc);
25213 %}
25214 ins_pipe(pipe_slow);
25215 %}
25216
25217 instruct vector_binOps_HF_reg(vec dst, vec src1, vec src2)
25218 %{
25219 match(Set dst (AddVHF src1 src2));
25220 match(Set dst (DivVHF src1 src2));
25221 match(Set dst (MulVHF src1 src2));
25222 match(Set dst (SubVHF src1 src2));
25223 format %{ "vector_binop_fp16 $dst, $src1, $src2" %}
25224 ins_encode %{
25225 int vlen_enc = vector_length_encoding(this);
25226 int opcode = this->ideal_Opcode();
25227 __ evfp16ph(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
25228 %}
25229 ins_pipe(pipe_slow);
25230 %}
25231
25232
25233 instruct vector_binOps_HF_mem(vec dst, vec src1, memory src2)
25234 %{
25235 match(Set dst (AddVHF src1 (VectorReinterpret (LoadVector src2))));
25236 match(Set dst (DivVHF src1 (VectorReinterpret (LoadVector src2))));
25237 match(Set dst (MulVHF src1 (VectorReinterpret (LoadVector src2))));
25238 match(Set dst (SubVHF src1 (VectorReinterpret (LoadVector src2))));
25239 format %{ "vector_binop_fp16_mem $dst, $src1, $src2" %}
25240 ins_encode %{
25241 int vlen_enc = vector_length_encoding(this);
25242 int opcode = this->ideal_Opcode();
25243 __ evfp16ph(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address, vlen_enc);
25244 %}
25245 ins_pipe(pipe_slow);
25246 %}
25247
25248 instruct vector_fma_HF_reg(vec dst, vec src1, vec src2)
25249 %{
25250 match(Set dst (FmaVHF src2 (Binary dst src1)));
25251 format %{ "vector_fma_fp16 $dst, $src1, $src2\t# $dst = $dst * $src1 + $src2 fma packedH" %}
25252 ins_encode %{
25253 int vlen_enc = vector_length_encoding(this);
25254 __ evfmadd132ph($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vlen_enc);
25255 %}
25256 ins_pipe( pipe_slow );
25257 %}
25258
25259 instruct vector_fma_HF_mem(vec dst, memory src1, vec src2)
25260 %{
25261 match(Set dst (FmaVHF src2 (Binary dst (VectorReinterpret (LoadVector src1)))));
25262 format %{ "vector_fma_fp16_mem $dst, $src1, $src2\t# $dst = $dst * $src1 + $src2 fma packedH" %}
25263 ins_encode %{
25264 int vlen_enc = vector_length_encoding(this);
25265 __ evfmadd132ph($dst$$XMMRegister, $src2$$XMMRegister, $src1$$Address, vlen_enc);
25266 %}
25267 ins_pipe( pipe_slow );
25268 %}
25269
25270 instruct vector_minmax_HF_mem_avx10_2(vec dst, vec src1, memory src2)
25271 %{
25272 predicate(VM_Version::supports_avx10_2());
25273 match(Set dst (MinVHF src1 (VectorReinterpret (LoadVector src2))));
25274 match(Set dst (MaxVHF src1 (VectorReinterpret (LoadVector src2))));
25275 format %{ "vector_min_max_fp16_mem $dst, $src1, $src2" %}
25276 ins_encode %{
25277 int vlen_enc = vector_length_encoding(this);
25278 int opcode = this->ideal_Opcode();
25279 __ vminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address,
25280 k0, vlen_enc);
25281 %}
25282 ins_pipe( pipe_slow );
25283 %}
25284
25285 instruct vector_minmax_HF_reg_avx10_2(vec dst, vec src1, vec src2)
25286 %{
25287 predicate(VM_Version::supports_avx10_2());
25288 match(Set dst (MinVHF src1 src2));
25289 match(Set dst (MaxVHF src1 src2));
25290 format %{ "vector_min_max_fp16 $dst, $src1, $src2" %}
25291 ins_encode %{
25292 int vlen_enc = vector_length_encoding(this);
25293 int opcode = this->ideal_Opcode();
25294 __ vminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
25295 k0, vlen_enc);
25296 %}
25297 ins_pipe( pipe_slow );
25298 %}
25299
25300 instruct vector_minmax_HF_reg(vec dst, vec src1, vec src2, kReg ktmp, vec xtmp1, vec xtmp2)
25301 %{
25302 predicate(!VM_Version::supports_avx10_2());
25303 match(Set dst (MinVHF src1 src2));
25304 match(Set dst (MaxVHF src1 src2));
25305 effect(TEMP_DEF dst, TEMP ktmp, TEMP xtmp1, TEMP xtmp2);
25306 format %{ "vector_min_max_fp16 $dst, $src1, $src2\t using $ktmp, $xtmp1 and $xtmp2 as TEMP" %}
25307 ins_encode %{
25308 int vlen_enc = vector_length_encoding(this);
25309 int opcode = this->ideal_Opcode();
25310 __ vminmax_fp16(opcode, $dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, $ktmp$$KRegister,
25311 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
25312 %}
25313 ins_pipe( pipe_slow );
25314 %}
25315
25316 //----------PEEPHOLE RULES-----------------------------------------------------
25317 // These must follow all instruction definitions as they use the names
25318 // defined in the instructions definitions.
25319 //
25320 // peeppredicate ( rule_predicate );
25321 // // the predicate unless which the peephole rule will be ignored
25322 //
25323 // peepmatch ( root_instr_name [preceding_instruction]* );
25324 //
25325 // peepprocedure ( procedure_name );
25326 // // provide a procedure name to perform the optimization, the procedure should
25327 // // reside in the architecture dependent peephole file, the method has the
25328 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
25329 // // with the arguments being the basic block, the current node index inside the
25330 // // block, the register allocator, the functions upon invoked return a new node
25331 // // defined in peepreplace, and the rules of the nodes appearing in the
25332 // // corresponding peepmatch, the function return true if successful, else
25333 // // return false
25334 //
25335 // peepconstraint %{
25336 // (instruction_number.operand_name relational_op instruction_number.operand_name
25337 // [, ...] );
25338 // // instruction numbers are zero-based using left to right order in peepmatch
25339 //
25340 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
25341 // // provide an instruction_number.operand_name for each operand that appears
25342 // // in the replacement instruction's match rule
25343 //
25344 // ---------VM FLAGS---------------------------------------------------------
25345 //
25346 // All peephole optimizations can be turned off using -XX:-OptoPeephole
25347 //
25348 // Each peephole rule is given an identifying number starting with zero and
25349 // increasing by one in the order seen by the parser. An individual peephole
25350 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
25351 // on the command-line.
25352 //
25353 // ---------CURRENT LIMITATIONS----------------------------------------------
25354 //
25355 // Only transformations inside a basic block (do we need more for peephole)
25356 //
25357 // ---------EXAMPLE----------------------------------------------------------
25358 //
25359 // // pertinent parts of existing instructions in architecture description
25360 // instruct movI(rRegI dst, rRegI src)
25361 // %{
25362 // match(Set dst (CopyI src));
25363 // %}
25364 //
25365 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
25366 // %{
25367 // match(Set dst (AddI dst src));
25368 // effect(KILL cr);
25369 // %}
25370 //
25371 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
25372 // %{
25373 // match(Set dst (AddI dst src));
25374 // %}
25375 //
25376 // 1. Simple replacement
25377 // - Only match adjacent instructions in same basic block
25378 // - Only equality constraints
25379 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
25380 // - Only one replacement instruction
25381 //
25382 // // Change (inc mov) to lea
25383 // peephole %{
25384 // // lea should only be emitted when beneficial
25385 // peeppredicate( VM_Version::supports_fast_2op_lea() );
25386 // // increment preceded by register-register move
25387 // peepmatch ( incI_rReg movI );
25388 // // require that the destination register of the increment
25389 // // match the destination register of the move
25390 // peepconstraint ( 0.dst == 1.dst );
25391 // // construct a replacement instruction that sets
25392 // // the destination to ( move's source register + one )
25393 // peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
25394 // %}
25395 //
25396 // 2. Procedural replacement
25397 // - More flexible finding relevent nodes
25398 // - More flexible constraints
25399 // - More flexible transformations
25400 // - May utilise architecture-dependent API more effectively
25401 // - Currently only one replacement instruction due to adlc parsing capabilities
25402 //
25403 // // Change (inc mov) to lea
25404 // peephole %{
25405 // // lea should only be emitted when beneficial
25406 // peeppredicate( VM_Version::supports_fast_2op_lea() );
25407 // // the rule numbers of these nodes inside are passed into the function below
25408 // peepmatch ( incI_rReg movI );
25409 // // the method that takes the responsibility of transformation
25410 // peepprocedure ( inc_mov_to_lea );
25411 // // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
25412 // // node is passed into the function above
25413 // peepreplace ( leaI_rReg_immI() );
25414 // %}
25415
25416 // These instructions is not matched by the matcher but used by the peephole
25417 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
25418 %{
25419 predicate(false);
25420 match(Set dst (AddI src1 src2));
25421 format %{ "leal $dst, [$src1 + $src2]" %}
25422 ins_encode %{
25423 Register dst = $dst$$Register;
25424 Register src1 = $src1$$Register;
25425 Register src2 = $src2$$Register;
25426 if (src1 != rbp && src1 != r13) {
25427 __ leal(dst, Address(src1, src2, Address::times_1));
25428 } else {
25429 assert(src2 != rbp && src2 != r13, "");
25430 __ leal(dst, Address(src2, src1, Address::times_1));
25431 }
25432 %}
25433 ins_pipe(ialu_reg_reg);
25434 %}
25435
25436 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
25437 %{
25438 predicate(false);
25439 match(Set dst (AddI src1 src2));
25440 format %{ "leal $dst, [$src1 + $src2]" %}
25441 ins_encode %{
25442 __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
25443 %}
25444 ins_pipe(ialu_reg_reg);
25445 %}
25446
25447 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
25448 %{
25449 predicate(false);
25450 match(Set dst (LShiftI src shift));
25451 format %{ "leal $dst, [$src << $shift]" %}
25452 ins_encode %{
25453 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
25454 Register src = $src$$Register;
25455 if (scale == Address::times_2 && src != rbp && src != r13) {
25456 __ leal($dst$$Register, Address(src, src, Address::times_1));
25457 } else {
25458 __ leal($dst$$Register, Address(noreg, src, scale));
25459 }
25460 %}
25461 ins_pipe(ialu_reg_reg);
25462 %}
25463
25464 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
25465 %{
25466 predicate(false);
25467 match(Set dst (AddL src1 src2));
25468 format %{ "leaq $dst, [$src1 + $src2]" %}
25469 ins_encode %{
25470 Register dst = $dst$$Register;
25471 Register src1 = $src1$$Register;
25472 Register src2 = $src2$$Register;
25473 if (src1 != rbp && src1 != r13) {
25474 __ leaq(dst, Address(src1, src2, Address::times_1));
25475 } else {
25476 assert(src2 != rbp && src2 != r13, "");
25477 __ leaq(dst, Address(src2, src1, Address::times_1));
25478 }
25479 %}
25480 ins_pipe(ialu_reg_reg);
25481 %}
25482
25483 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
25484 %{
25485 predicate(false);
25486 match(Set dst (AddL src1 src2));
25487 format %{ "leaq $dst, [$src1 + $src2]" %}
25488 ins_encode %{
25489 __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
25490 %}
25491 ins_pipe(ialu_reg_reg);
25492 %}
25493
25494 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
25495 %{
25496 predicate(false);
25497 match(Set dst (LShiftL src shift));
25498 format %{ "leaq $dst, [$src << $shift]" %}
25499 ins_encode %{
25500 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
25501 Register src = $src$$Register;
25502 if (scale == Address::times_2 && src != rbp && src != r13) {
25503 __ leaq($dst$$Register, Address(src, src, Address::times_1));
25504 } else {
25505 __ leaq($dst$$Register, Address(noreg, src, scale));
25506 }
25507 %}
25508 ins_pipe(ialu_reg_reg);
25509 %}
25510
25511 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
25512 // sal}) with lea instructions. The {add, sal} rules are beneficial in
25513 // processors with at least partial ALU support for lea
25514 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
25515 // beneficial for processors with full ALU support
25516 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
25517
25518 peephole
25519 %{
25520 peeppredicate(VM_Version::supports_fast_2op_lea());
25521 peepmatch (addI_rReg);
25522 peepprocedure (lea_coalesce_reg);
25523 peepreplace (leaI_rReg_rReg_peep());
25524 %}
25525
25526 peephole
25527 %{
25528 peeppredicate(VM_Version::supports_fast_2op_lea());
25529 peepmatch (addI_rReg_imm);
25530 peepprocedure (lea_coalesce_imm);
25531 peepreplace (leaI_rReg_immI_peep());
25532 %}
25533
25534 peephole
25535 %{
25536 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25537 VM_Version::is_intel_cascade_lake());
25538 peepmatch (incI_rReg);
25539 peepprocedure (lea_coalesce_imm);
25540 peepreplace (leaI_rReg_immI_peep());
25541 %}
25542
25543 peephole
25544 %{
25545 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25546 VM_Version::is_intel_cascade_lake());
25547 peepmatch (decI_rReg);
25548 peepprocedure (lea_coalesce_imm);
25549 peepreplace (leaI_rReg_immI_peep());
25550 %}
25551
25552 peephole
25553 %{
25554 peeppredicate(VM_Version::supports_fast_2op_lea());
25555 peepmatch (salI_rReg_immI2);
25556 peepprocedure (lea_coalesce_imm);
25557 peepreplace (leaI_rReg_immI2_peep());
25558 %}
25559
25560 peephole
25561 %{
25562 peeppredicate(VM_Version::supports_fast_2op_lea());
25563 peepmatch (addL_rReg);
25564 peepprocedure (lea_coalesce_reg);
25565 peepreplace (leaL_rReg_rReg_peep());
25566 %}
25567
25568 peephole
25569 %{
25570 peeppredicate(VM_Version::supports_fast_2op_lea());
25571 peepmatch (addL_rReg_imm);
25572 peepprocedure (lea_coalesce_imm);
25573 peepreplace (leaL_rReg_immL32_peep());
25574 %}
25575
25576 peephole
25577 %{
25578 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25579 VM_Version::is_intel_cascade_lake());
25580 peepmatch (incL_rReg);
25581 peepprocedure (lea_coalesce_imm);
25582 peepreplace (leaL_rReg_immL32_peep());
25583 %}
25584
25585 peephole
25586 %{
25587 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25588 VM_Version::is_intel_cascade_lake());
25589 peepmatch (decL_rReg);
25590 peepprocedure (lea_coalesce_imm);
25591 peepreplace (leaL_rReg_immL32_peep());
25592 %}
25593
25594 peephole
25595 %{
25596 peeppredicate(VM_Version::supports_fast_2op_lea());
25597 peepmatch (salL_rReg_immI2);
25598 peepprocedure (lea_coalesce_imm);
25599 peepreplace (leaL_rReg_immI2_peep());
25600 %}
25601
25602 peephole
25603 %{
25604 peepmatch (leaPCompressedOopOffset);
25605 peepprocedure (lea_remove_redundant);
25606 %}
25607
25608 peephole
25609 %{
25610 peepmatch (leaP8Narrow);
25611 peepprocedure (lea_remove_redundant);
25612 %}
25613
25614 peephole
25615 %{
25616 peepmatch (leaP32Narrow);
25617 peepprocedure (lea_remove_redundant);
25618 %}
25619
25620 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
25621 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
25622
25623 //int variant
25624 peephole
25625 %{
25626 peepmatch (testI_reg);
25627 peepprocedure (test_may_remove);
25628 %}
25629
25630 //long variant
25631 peephole
25632 %{
25633 peepmatch (testL_reg);
25634 peepprocedure (test_may_remove);
25635 %}
25636
25637
25638 //----------SMARTSPILL RULES---------------------------------------------------
25639 // These must follow all instruction definitions as they use the names
25640 // defined in the instructions definitions.