1 //
2 // Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23 //
24
25 // X86 AMD64 Architecture Description File
26
27 //----------REGISTER DEFINITION BLOCK------------------------------------------
28 // This information is used by the matcher and the register allocator to
29 // describe individual registers and classes of registers within the target
30 // architecture.
31
32 register %{
33 //----------Architecture Description Register Definitions----------------------
34 // General Registers
35 // "reg_def" name ( register save type, C convention save type,
36 // ideal register type, encoding );
37 // Register Save Types:
38 //
39 // NS = No-Save: The register allocator assumes that these registers
40 // can be used without saving upon entry to the method, &
41 // that they do not need to be saved at call sites.
42 //
43 // SOC = Save-On-Call: The register allocator assumes that these registers
44 // can be used without saving upon entry to the method,
45 // but that they must be saved at call sites.
46 //
47 // SOE = Save-On-Entry: The register allocator assumes that these registers
48 // must be saved before using them upon entry to the
49 // method, but they do not need to be saved at call
50 // sites.
51 //
52 // AS = Always-Save: The register allocator assumes that these registers
53 // must be saved before using them upon entry to the
54 // method, & that they must be saved at call sites.
55 //
56 // Ideal Register Type is used to determine how to save & restore a
57 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
58 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
59 //
60 // The encoding number is the actual bit-pattern placed into the opcodes.
61
62 // General Registers
63 // R8-R15 must be encoded with REX. (RSP, RBP, RSI, RDI need REX when
64 // used as byte registers)
65
66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
69
70 reg_def RAX (SOC, SOC, Op_RegI, 0, rax->as_VMReg());
71 reg_def RAX_H(SOC, SOC, Op_RegI, 0, rax->as_VMReg()->next());
72
73 reg_def RCX (SOC, SOC, Op_RegI, 1, rcx->as_VMReg());
74 reg_def RCX_H(SOC, SOC, Op_RegI, 1, rcx->as_VMReg()->next());
75
76 reg_def RDX (SOC, SOC, Op_RegI, 2, rdx->as_VMReg());
77 reg_def RDX_H(SOC, SOC, Op_RegI, 2, rdx->as_VMReg()->next());
78
79 reg_def RBX (SOC, SOE, Op_RegI, 3, rbx->as_VMReg());
80 reg_def RBX_H(SOC, SOE, Op_RegI, 3, rbx->as_VMReg()->next());
81
82 reg_def RSP (NS, NS, Op_RegI, 4, rsp->as_VMReg());
83 reg_def RSP_H(NS, NS, Op_RegI, 4, rsp->as_VMReg()->next());
84
85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
86 reg_def RBP (NS, SOE, Op_RegI, 5, rbp->as_VMReg());
87 reg_def RBP_H(NS, SOE, Op_RegI, 5, rbp->as_VMReg()->next());
88
89 #ifdef _WIN64
90
91 reg_def RSI (SOC, SOE, Op_RegI, 6, rsi->as_VMReg());
92 reg_def RSI_H(SOC, SOE, Op_RegI, 6, rsi->as_VMReg()->next());
93
94 reg_def RDI (SOC, SOE, Op_RegI, 7, rdi->as_VMReg());
95 reg_def RDI_H(SOC, SOE, Op_RegI, 7, rdi->as_VMReg()->next());
96
97 #else
98
99 reg_def RSI (SOC, SOC, Op_RegI, 6, rsi->as_VMReg());
100 reg_def RSI_H(SOC, SOC, Op_RegI, 6, rsi->as_VMReg()->next());
101
102 reg_def RDI (SOC, SOC, Op_RegI, 7, rdi->as_VMReg());
103 reg_def RDI_H(SOC, SOC, Op_RegI, 7, rdi->as_VMReg()->next());
104
105 #endif
106
107 reg_def R8 (SOC, SOC, Op_RegI, 8, r8->as_VMReg());
108 reg_def R8_H (SOC, SOC, Op_RegI, 8, r8->as_VMReg()->next());
109
110 reg_def R9 (SOC, SOC, Op_RegI, 9, r9->as_VMReg());
111 reg_def R9_H (SOC, SOC, Op_RegI, 9, r9->as_VMReg()->next());
112
113 reg_def R10 (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
115
116 reg_def R11 (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
118
119 reg_def R12 (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
121
122 reg_def R13 (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
124
125 reg_def R14 (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
127
128 reg_def R15 (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
130
131 reg_def R16 (SOC, SOC, Op_RegI, 16, r16->as_VMReg());
132 reg_def R16_H(SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
133
134 reg_def R17 (SOC, SOC, Op_RegI, 17, r17->as_VMReg());
135 reg_def R17_H(SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
136
137 reg_def R18 (SOC, SOC, Op_RegI, 18, r18->as_VMReg());
138 reg_def R18_H(SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
139
140 reg_def R19 (SOC, SOC, Op_RegI, 19, r19->as_VMReg());
141 reg_def R19_H(SOC, SOC, Op_RegI, 19, r19->as_VMReg()->next());
142
143 reg_def R20 (SOC, SOC, Op_RegI, 20, r20->as_VMReg());
144 reg_def R20_H(SOC, SOC, Op_RegI, 20, r20->as_VMReg()->next());
145
146 reg_def R21 (SOC, SOC, Op_RegI, 21, r21->as_VMReg());
147 reg_def R21_H(SOC, SOC, Op_RegI, 21, r21->as_VMReg()->next());
148
149 reg_def R22 (SOC, SOC, Op_RegI, 22, r22->as_VMReg());
150 reg_def R22_H(SOC, SOC, Op_RegI, 22, r22->as_VMReg()->next());
151
152 reg_def R23 (SOC, SOC, Op_RegI, 23, r23->as_VMReg());
153 reg_def R23_H(SOC, SOC, Op_RegI, 23, r23->as_VMReg()->next());
154
155 reg_def R24 (SOC, SOC, Op_RegI, 24, r24->as_VMReg());
156 reg_def R24_H(SOC, SOC, Op_RegI, 24, r24->as_VMReg()->next());
157
158 reg_def R25 (SOC, SOC, Op_RegI, 25, r25->as_VMReg());
159 reg_def R25_H(SOC, SOC, Op_RegI, 25, r25->as_VMReg()->next());
160
161 reg_def R26 (SOC, SOC, Op_RegI, 26, r26->as_VMReg());
162 reg_def R26_H(SOC, SOC, Op_RegI, 26, r26->as_VMReg()->next());
163
164 reg_def R27 (SOC, SOC, Op_RegI, 27, r27->as_VMReg());
165 reg_def R27_H(SOC, SOC, Op_RegI, 27, r27->as_VMReg()->next());
166
167 reg_def R28 (SOC, SOC, Op_RegI, 28, r28->as_VMReg());
168 reg_def R28_H(SOC, SOC, Op_RegI, 28, r28->as_VMReg()->next());
169
170 reg_def R29 (SOC, SOC, Op_RegI, 29, r29->as_VMReg());
171 reg_def R29_H(SOC, SOC, Op_RegI, 29, r29->as_VMReg()->next());
172
173 reg_def R30 (SOC, SOC, Op_RegI, 30, r30->as_VMReg());
174 reg_def R30_H(SOC, SOC, Op_RegI, 30, r30->as_VMReg()->next());
175
176 reg_def R31 (SOC, SOC, Op_RegI, 31, r31->as_VMReg());
177 reg_def R31_H(SOC, SOC, Op_RegI, 31, r31->as_VMReg()->next());
178
179 // Floating Point Registers
180
181 // Specify priority of register selection within phases of register
182 // allocation. Highest priority is first. A useful heuristic is to
183 // give registers a low priority when they are required by machine
184 // instructions, like EAX and EDX on I486, and choose no-save registers
185 // before save-on-call, & save-on-call before save-on-entry. Registers
186 // which participate in fixed calling sequences should come last.
187 // Registers which are used as pairs must fall on an even boundary.
188
189 alloc_class chunk0(R10, R10_H,
190 R11, R11_H,
191 R8, R8_H,
192 R9, R9_H,
193 R12, R12_H,
194 RCX, RCX_H,
195 RBX, RBX_H,
196 RDI, RDI_H,
197 RDX, RDX_H,
198 RSI, RSI_H,
199 RAX, RAX_H,
200 RBP, RBP_H,
201 R13, R13_H,
202 R14, R14_H,
203 R15, R15_H,
204 R16, R16_H,
205 R17, R17_H,
206 R18, R18_H,
207 R19, R19_H,
208 R20, R20_H,
209 R21, R21_H,
210 R22, R22_H,
211 R23, R23_H,
212 R24, R24_H,
213 R25, R25_H,
214 R26, R26_H,
215 R27, R27_H,
216 R28, R28_H,
217 R29, R29_H,
218 R30, R30_H,
219 R31, R31_H,
220 RSP, RSP_H);
221
222 // XMM registers. 512-bit registers or 8 words each, labeled (a)-p.
223 // Word a in each register holds a Float, words ab hold a Double.
224 // The whole registers are used in SSE4.2 version intrinsics,
225 // array copy stubs and superword operations (see UseSSE42Intrinsics,
226 // UseXMMForArrayCopy and UseSuperword flags).
227 // For pre EVEX enabled architectures:
228 // XMM8-XMM15 must be encoded with REX (VEX for UseAVX)
229 // For EVEX enabled architectures:
230 // XMM8-XMM31 must be encoded with REX (EVEX for UseAVX).
231 //
232 // Linux ABI: No register preserved across function calls
233 // XMM0-XMM7 might hold parameters
234 // Windows ABI: XMM6-XMM15 preserved across function calls
235 // XMM0-XMM3 might hold parameters
236
237 reg_def XMM0 ( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg());
238 reg_def XMM0b( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(1));
239 reg_def XMM0c( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(2));
240 reg_def XMM0d( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(3));
241 reg_def XMM0e( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(4));
242 reg_def XMM0f( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(5));
243 reg_def XMM0g( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(6));
244 reg_def XMM0h( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(7));
245 reg_def XMM0i( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(8));
246 reg_def XMM0j( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(9));
247 reg_def XMM0k( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(10));
248 reg_def XMM0l( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(11));
249 reg_def XMM0m( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(12));
250 reg_def XMM0n( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(13));
251 reg_def XMM0o( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(14));
252 reg_def XMM0p( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(15));
253
254 reg_def XMM1 ( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg());
255 reg_def XMM1b( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(1));
256 reg_def XMM1c( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(2));
257 reg_def XMM1d( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(3));
258 reg_def XMM1e( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(4));
259 reg_def XMM1f( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(5));
260 reg_def XMM1g( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(6));
261 reg_def XMM1h( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(7));
262 reg_def XMM1i( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(8));
263 reg_def XMM1j( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(9));
264 reg_def XMM1k( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(10));
265 reg_def XMM1l( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(11));
266 reg_def XMM1m( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(12));
267 reg_def XMM1n( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(13));
268 reg_def XMM1o( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(14));
269 reg_def XMM1p( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(15));
270
271 reg_def XMM2 ( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg());
272 reg_def XMM2b( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(1));
273 reg_def XMM2c( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(2));
274 reg_def XMM2d( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(3));
275 reg_def XMM2e( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(4));
276 reg_def XMM2f( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(5));
277 reg_def XMM2g( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(6));
278 reg_def XMM2h( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(7));
279 reg_def XMM2i( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(8));
280 reg_def XMM2j( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(9));
281 reg_def XMM2k( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(10));
282 reg_def XMM2l( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(11));
283 reg_def XMM2m( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(12));
284 reg_def XMM2n( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(13));
285 reg_def XMM2o( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(14));
286 reg_def XMM2p( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(15));
287
288 reg_def XMM3 ( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg());
289 reg_def XMM3b( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(1));
290 reg_def XMM3c( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(2));
291 reg_def XMM3d( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(3));
292 reg_def XMM3e( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(4));
293 reg_def XMM3f( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(5));
294 reg_def XMM3g( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(6));
295 reg_def XMM3h( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(7));
296 reg_def XMM3i( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(8));
297 reg_def XMM3j( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(9));
298 reg_def XMM3k( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(10));
299 reg_def XMM3l( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(11));
300 reg_def XMM3m( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(12));
301 reg_def XMM3n( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(13));
302 reg_def XMM3o( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(14));
303 reg_def XMM3p( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(15));
304
305 reg_def XMM4 ( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg());
306 reg_def XMM4b( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(1));
307 reg_def XMM4c( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(2));
308 reg_def XMM4d( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(3));
309 reg_def XMM4e( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(4));
310 reg_def XMM4f( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(5));
311 reg_def XMM4g( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(6));
312 reg_def XMM4h( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(7));
313 reg_def XMM4i( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(8));
314 reg_def XMM4j( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(9));
315 reg_def XMM4k( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(10));
316 reg_def XMM4l( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(11));
317 reg_def XMM4m( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(12));
318 reg_def XMM4n( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(13));
319 reg_def XMM4o( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(14));
320 reg_def XMM4p( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(15));
321
322 reg_def XMM5 ( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg());
323 reg_def XMM5b( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(1));
324 reg_def XMM5c( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(2));
325 reg_def XMM5d( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(3));
326 reg_def XMM5e( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(4));
327 reg_def XMM5f( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(5));
328 reg_def XMM5g( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(6));
329 reg_def XMM5h( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(7));
330 reg_def XMM5i( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(8));
331 reg_def XMM5j( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(9));
332 reg_def XMM5k( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(10));
333 reg_def XMM5l( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(11));
334 reg_def XMM5m( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(12));
335 reg_def XMM5n( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(13));
336 reg_def XMM5o( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(14));
337 reg_def XMM5p( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(15));
338
339 reg_def XMM6 ( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg());
340 reg_def XMM6b( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(1));
341 reg_def XMM6c( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(2));
342 reg_def XMM6d( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(3));
343 reg_def XMM6e( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(4));
344 reg_def XMM6f( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(5));
345 reg_def XMM6g( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(6));
346 reg_def XMM6h( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(7));
347 reg_def XMM6i( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(8));
348 reg_def XMM6j( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(9));
349 reg_def XMM6k( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(10));
350 reg_def XMM6l( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(11));
351 reg_def XMM6m( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(12));
352 reg_def XMM6n( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(13));
353 reg_def XMM6o( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(14));
354 reg_def XMM6p( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(15));
355
356 reg_def XMM7 ( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg());
357 reg_def XMM7b( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(1));
358 reg_def XMM7c( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(2));
359 reg_def XMM7d( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(3));
360 reg_def XMM7e( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(4));
361 reg_def XMM7f( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(5));
362 reg_def XMM7g( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(6));
363 reg_def XMM7h( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(7));
364 reg_def XMM7i( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(8));
365 reg_def XMM7j( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(9));
366 reg_def XMM7k( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(10));
367 reg_def XMM7l( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(11));
368 reg_def XMM7m( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(12));
369 reg_def XMM7n( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(13));
370 reg_def XMM7o( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(14));
371 reg_def XMM7p( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(15));
372
373 reg_def XMM8 ( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg());
374 reg_def XMM8b( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(1));
375 reg_def XMM8c( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(2));
376 reg_def XMM8d( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(3));
377 reg_def XMM8e( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(4));
378 reg_def XMM8f( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(5));
379 reg_def XMM8g( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(6));
380 reg_def XMM8h( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(7));
381 reg_def XMM8i( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(8));
382 reg_def XMM8j( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(9));
383 reg_def XMM8k( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(10));
384 reg_def XMM8l( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(11));
385 reg_def XMM8m( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(12));
386 reg_def XMM8n( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(13));
387 reg_def XMM8o( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(14));
388 reg_def XMM8p( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(15));
389
390 reg_def XMM9 ( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg());
391 reg_def XMM9b( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(1));
392 reg_def XMM9c( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(2));
393 reg_def XMM9d( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(3));
394 reg_def XMM9e( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(4));
395 reg_def XMM9f( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(5));
396 reg_def XMM9g( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(6));
397 reg_def XMM9h( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(7));
398 reg_def XMM9i( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(8));
399 reg_def XMM9j( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(9));
400 reg_def XMM9k( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(10));
401 reg_def XMM9l( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(11));
402 reg_def XMM9m( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(12));
403 reg_def XMM9n( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(13));
404 reg_def XMM9o( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(14));
405 reg_def XMM9p( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(15));
406
407 reg_def XMM10 ( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg());
408 reg_def XMM10b( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(1));
409 reg_def XMM10c( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(2));
410 reg_def XMM10d( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(3));
411 reg_def XMM10e( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(4));
412 reg_def XMM10f( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(5));
413 reg_def XMM10g( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(6));
414 reg_def XMM10h( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(7));
415 reg_def XMM10i( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(8));
416 reg_def XMM10j( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(9));
417 reg_def XMM10k( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(10));
418 reg_def XMM10l( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(11));
419 reg_def XMM10m( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(12));
420 reg_def XMM10n( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(13));
421 reg_def XMM10o( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(14));
422 reg_def XMM10p( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(15));
423
424 reg_def XMM11 ( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg());
425 reg_def XMM11b( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(1));
426 reg_def XMM11c( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(2));
427 reg_def XMM11d( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(3));
428 reg_def XMM11e( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(4));
429 reg_def XMM11f( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(5));
430 reg_def XMM11g( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(6));
431 reg_def XMM11h( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(7));
432 reg_def XMM11i( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(8));
433 reg_def XMM11j( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(9));
434 reg_def XMM11k( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(10));
435 reg_def XMM11l( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(11));
436 reg_def XMM11m( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(12));
437 reg_def XMM11n( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(13));
438 reg_def XMM11o( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(14));
439 reg_def XMM11p( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(15));
440
441 reg_def XMM12 ( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg());
442 reg_def XMM12b( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(1));
443 reg_def XMM12c( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(2));
444 reg_def XMM12d( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(3));
445 reg_def XMM12e( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(4));
446 reg_def XMM12f( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(5));
447 reg_def XMM12g( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(6));
448 reg_def XMM12h( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(7));
449 reg_def XMM12i( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(8));
450 reg_def XMM12j( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(9));
451 reg_def XMM12k( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(10));
452 reg_def XMM12l( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(11));
453 reg_def XMM12m( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(12));
454 reg_def XMM12n( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(13));
455 reg_def XMM12o( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(14));
456 reg_def XMM12p( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(15));
457
458 reg_def XMM13 ( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg());
459 reg_def XMM13b( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(1));
460 reg_def XMM13c( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(2));
461 reg_def XMM13d( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(3));
462 reg_def XMM13e( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(4));
463 reg_def XMM13f( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(5));
464 reg_def XMM13g( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(6));
465 reg_def XMM13h( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(7));
466 reg_def XMM13i( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(8));
467 reg_def XMM13j( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(9));
468 reg_def XMM13k( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(10));
469 reg_def XMM13l( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(11));
470 reg_def XMM13m( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(12));
471 reg_def XMM13n( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(13));
472 reg_def XMM13o( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(14));
473 reg_def XMM13p( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(15));
474
475 reg_def XMM14 ( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg());
476 reg_def XMM14b( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(1));
477 reg_def XMM14c( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(2));
478 reg_def XMM14d( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(3));
479 reg_def XMM14e( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(4));
480 reg_def XMM14f( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(5));
481 reg_def XMM14g( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(6));
482 reg_def XMM14h( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(7));
483 reg_def XMM14i( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(8));
484 reg_def XMM14j( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(9));
485 reg_def XMM14k( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(10));
486 reg_def XMM14l( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(11));
487 reg_def XMM14m( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(12));
488 reg_def XMM14n( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(13));
489 reg_def XMM14o( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(14));
490 reg_def XMM14p( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(15));
491
492 reg_def XMM15 ( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg());
493 reg_def XMM15b( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(1));
494 reg_def XMM15c( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(2));
495 reg_def XMM15d( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(3));
496 reg_def XMM15e( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(4));
497 reg_def XMM15f( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(5));
498 reg_def XMM15g( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(6));
499 reg_def XMM15h( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(7));
500 reg_def XMM15i( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(8));
501 reg_def XMM15j( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(9));
502 reg_def XMM15k( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(10));
503 reg_def XMM15l( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(11));
504 reg_def XMM15m( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(12));
505 reg_def XMM15n( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(13));
506 reg_def XMM15o( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(14));
507 reg_def XMM15p( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(15));
508
509 reg_def XMM16 ( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg());
510 reg_def XMM16b( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(1));
511 reg_def XMM16c( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(2));
512 reg_def XMM16d( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(3));
513 reg_def XMM16e( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(4));
514 reg_def XMM16f( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(5));
515 reg_def XMM16g( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(6));
516 reg_def XMM16h( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(7));
517 reg_def XMM16i( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(8));
518 reg_def XMM16j( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(9));
519 reg_def XMM16k( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(10));
520 reg_def XMM16l( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(11));
521 reg_def XMM16m( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(12));
522 reg_def XMM16n( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(13));
523 reg_def XMM16o( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(14));
524 reg_def XMM16p( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(15));
525
526 reg_def XMM17 ( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg());
527 reg_def XMM17b( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(1));
528 reg_def XMM17c( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(2));
529 reg_def XMM17d( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(3));
530 reg_def XMM17e( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(4));
531 reg_def XMM17f( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(5));
532 reg_def XMM17g( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(6));
533 reg_def XMM17h( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(7));
534 reg_def XMM17i( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(8));
535 reg_def XMM17j( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(9));
536 reg_def XMM17k( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(10));
537 reg_def XMM17l( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(11));
538 reg_def XMM17m( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(12));
539 reg_def XMM17n( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(13));
540 reg_def XMM17o( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(14));
541 reg_def XMM17p( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(15));
542
543 reg_def XMM18 ( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg());
544 reg_def XMM18b( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(1));
545 reg_def XMM18c( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(2));
546 reg_def XMM18d( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(3));
547 reg_def XMM18e( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(4));
548 reg_def XMM18f( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(5));
549 reg_def XMM18g( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(6));
550 reg_def XMM18h( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(7));
551 reg_def XMM18i( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(8));
552 reg_def XMM18j( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(9));
553 reg_def XMM18k( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(10));
554 reg_def XMM18l( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(11));
555 reg_def XMM18m( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(12));
556 reg_def XMM18n( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(13));
557 reg_def XMM18o( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(14));
558 reg_def XMM18p( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(15));
559
560 reg_def XMM19 ( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg());
561 reg_def XMM19b( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(1));
562 reg_def XMM19c( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(2));
563 reg_def XMM19d( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(3));
564 reg_def XMM19e( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(4));
565 reg_def XMM19f( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(5));
566 reg_def XMM19g( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(6));
567 reg_def XMM19h( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(7));
568 reg_def XMM19i( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(8));
569 reg_def XMM19j( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(9));
570 reg_def XMM19k( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(10));
571 reg_def XMM19l( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(11));
572 reg_def XMM19m( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(12));
573 reg_def XMM19n( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(13));
574 reg_def XMM19o( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(14));
575 reg_def XMM19p( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(15));
576
577 reg_def XMM20 ( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg());
578 reg_def XMM20b( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(1));
579 reg_def XMM20c( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(2));
580 reg_def XMM20d( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(3));
581 reg_def XMM20e( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(4));
582 reg_def XMM20f( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(5));
583 reg_def XMM20g( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(6));
584 reg_def XMM20h( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(7));
585 reg_def XMM20i( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(8));
586 reg_def XMM20j( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(9));
587 reg_def XMM20k( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(10));
588 reg_def XMM20l( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(11));
589 reg_def XMM20m( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(12));
590 reg_def XMM20n( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(13));
591 reg_def XMM20o( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(14));
592 reg_def XMM20p( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(15));
593
594 reg_def XMM21 ( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg());
595 reg_def XMM21b( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(1));
596 reg_def XMM21c( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(2));
597 reg_def XMM21d( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(3));
598 reg_def XMM21e( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(4));
599 reg_def XMM21f( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(5));
600 reg_def XMM21g( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(6));
601 reg_def XMM21h( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(7));
602 reg_def XMM21i( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(8));
603 reg_def XMM21j( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(9));
604 reg_def XMM21k( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(10));
605 reg_def XMM21l( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(11));
606 reg_def XMM21m( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(12));
607 reg_def XMM21n( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(13));
608 reg_def XMM21o( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(14));
609 reg_def XMM21p( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(15));
610
611 reg_def XMM22 ( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg());
612 reg_def XMM22b( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(1));
613 reg_def XMM22c( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(2));
614 reg_def XMM22d( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(3));
615 reg_def XMM22e( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(4));
616 reg_def XMM22f( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(5));
617 reg_def XMM22g( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(6));
618 reg_def XMM22h( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(7));
619 reg_def XMM22i( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(8));
620 reg_def XMM22j( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(9));
621 reg_def XMM22k( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(10));
622 reg_def XMM22l( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(11));
623 reg_def XMM22m( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(12));
624 reg_def XMM22n( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(13));
625 reg_def XMM22o( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(14));
626 reg_def XMM22p( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(15));
627
628 reg_def XMM23 ( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg());
629 reg_def XMM23b( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(1));
630 reg_def XMM23c( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(2));
631 reg_def XMM23d( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(3));
632 reg_def XMM23e( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(4));
633 reg_def XMM23f( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(5));
634 reg_def XMM23g( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(6));
635 reg_def XMM23h( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(7));
636 reg_def XMM23i( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(8));
637 reg_def XMM23j( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(9));
638 reg_def XMM23k( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(10));
639 reg_def XMM23l( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(11));
640 reg_def XMM23m( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(12));
641 reg_def XMM23n( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(13));
642 reg_def XMM23o( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(14));
643 reg_def XMM23p( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(15));
644
645 reg_def XMM24 ( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg());
646 reg_def XMM24b( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(1));
647 reg_def XMM24c( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(2));
648 reg_def XMM24d( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(3));
649 reg_def XMM24e( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(4));
650 reg_def XMM24f( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(5));
651 reg_def XMM24g( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(6));
652 reg_def XMM24h( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(7));
653 reg_def XMM24i( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(8));
654 reg_def XMM24j( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(9));
655 reg_def XMM24k( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(10));
656 reg_def XMM24l( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(11));
657 reg_def XMM24m( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(12));
658 reg_def XMM24n( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(13));
659 reg_def XMM24o( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(14));
660 reg_def XMM24p( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(15));
661
662 reg_def XMM25 ( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg());
663 reg_def XMM25b( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(1));
664 reg_def XMM25c( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(2));
665 reg_def XMM25d( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(3));
666 reg_def XMM25e( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(4));
667 reg_def XMM25f( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(5));
668 reg_def XMM25g( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(6));
669 reg_def XMM25h( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(7));
670 reg_def XMM25i( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(8));
671 reg_def XMM25j( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(9));
672 reg_def XMM25k( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(10));
673 reg_def XMM25l( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(11));
674 reg_def XMM25m( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(12));
675 reg_def XMM25n( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(13));
676 reg_def XMM25o( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(14));
677 reg_def XMM25p( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(15));
678
679 reg_def XMM26 ( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg());
680 reg_def XMM26b( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(1));
681 reg_def XMM26c( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(2));
682 reg_def XMM26d( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(3));
683 reg_def XMM26e( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(4));
684 reg_def XMM26f( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(5));
685 reg_def XMM26g( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(6));
686 reg_def XMM26h( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(7));
687 reg_def XMM26i( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(8));
688 reg_def XMM26j( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(9));
689 reg_def XMM26k( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(10));
690 reg_def XMM26l( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(11));
691 reg_def XMM26m( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(12));
692 reg_def XMM26n( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(13));
693 reg_def XMM26o( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(14));
694 reg_def XMM26p( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(15));
695
696 reg_def XMM27 ( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg());
697 reg_def XMM27b( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(1));
698 reg_def XMM27c( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(2));
699 reg_def XMM27d( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(3));
700 reg_def XMM27e( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(4));
701 reg_def XMM27f( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(5));
702 reg_def XMM27g( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(6));
703 reg_def XMM27h( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(7));
704 reg_def XMM27i( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(8));
705 reg_def XMM27j( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(9));
706 reg_def XMM27k( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(10));
707 reg_def XMM27l( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(11));
708 reg_def XMM27m( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(12));
709 reg_def XMM27n( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(13));
710 reg_def XMM27o( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(14));
711 reg_def XMM27p( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(15));
712
713 reg_def XMM28 ( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg());
714 reg_def XMM28b( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(1));
715 reg_def XMM28c( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(2));
716 reg_def XMM28d( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(3));
717 reg_def XMM28e( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(4));
718 reg_def XMM28f( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(5));
719 reg_def XMM28g( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(6));
720 reg_def XMM28h( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(7));
721 reg_def XMM28i( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(8));
722 reg_def XMM28j( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(9));
723 reg_def XMM28k( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(10));
724 reg_def XMM28l( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(11));
725 reg_def XMM28m( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(12));
726 reg_def XMM28n( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(13));
727 reg_def XMM28o( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(14));
728 reg_def XMM28p( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(15));
729
730 reg_def XMM29 ( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg());
731 reg_def XMM29b( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(1));
732 reg_def XMM29c( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(2));
733 reg_def XMM29d( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(3));
734 reg_def XMM29e( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(4));
735 reg_def XMM29f( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(5));
736 reg_def XMM29g( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(6));
737 reg_def XMM29h( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(7));
738 reg_def XMM29i( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(8));
739 reg_def XMM29j( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(9));
740 reg_def XMM29k( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(10));
741 reg_def XMM29l( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(11));
742 reg_def XMM29m( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(12));
743 reg_def XMM29n( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(13));
744 reg_def XMM29o( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(14));
745 reg_def XMM29p( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(15));
746
747 reg_def XMM30 ( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg());
748 reg_def XMM30b( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(1));
749 reg_def XMM30c( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(2));
750 reg_def XMM30d( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(3));
751 reg_def XMM30e( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(4));
752 reg_def XMM30f( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(5));
753 reg_def XMM30g( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(6));
754 reg_def XMM30h( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(7));
755 reg_def XMM30i( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(8));
756 reg_def XMM30j( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(9));
757 reg_def XMM30k( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(10));
758 reg_def XMM30l( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(11));
759 reg_def XMM30m( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(12));
760 reg_def XMM30n( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(13));
761 reg_def XMM30o( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(14));
762 reg_def XMM30p( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(15));
763
764 reg_def XMM31 ( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg());
765 reg_def XMM31b( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(1));
766 reg_def XMM31c( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(2));
767 reg_def XMM31d( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(3));
768 reg_def XMM31e( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(4));
769 reg_def XMM31f( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(5));
770 reg_def XMM31g( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(6));
771 reg_def XMM31h( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(7));
772 reg_def XMM31i( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(8));
773 reg_def XMM31j( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(9));
774 reg_def XMM31k( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(10));
775 reg_def XMM31l( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(11));
776 reg_def XMM31m( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(12));
777 reg_def XMM31n( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(13));
778 reg_def XMM31o( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(14));
779 reg_def XMM31p( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(15));
780
781 reg_def RFLAGS(SOC, SOC, 0, 16, VMRegImpl::Bad());
782
783 // AVX3 Mask Registers.
784 reg_def K1 (SOC, SOC, Op_RegI, 1, k1->as_VMReg());
785 reg_def K1_H (SOC, SOC, Op_RegI, 1, k1->as_VMReg()->next());
786
787 reg_def K2 (SOC, SOC, Op_RegI, 2, k2->as_VMReg());
788 reg_def K2_H (SOC, SOC, Op_RegI, 2, k2->as_VMReg()->next());
789
790 reg_def K3 (SOC, SOC, Op_RegI, 3, k3->as_VMReg());
791 reg_def K3_H (SOC, SOC, Op_RegI, 3, k3->as_VMReg()->next());
792
793 reg_def K4 (SOC, SOC, Op_RegI, 4, k4->as_VMReg());
794 reg_def K4_H (SOC, SOC, Op_RegI, 4, k4->as_VMReg()->next());
795
796 reg_def K5 (SOC, SOC, Op_RegI, 5, k5->as_VMReg());
797 reg_def K5_H (SOC, SOC, Op_RegI, 5, k5->as_VMReg()->next());
798
799 reg_def K6 (SOC, SOC, Op_RegI, 6, k6->as_VMReg());
800 reg_def K6_H (SOC, SOC, Op_RegI, 6, k6->as_VMReg()->next());
801
802 reg_def K7 (SOC, SOC, Op_RegI, 7, k7->as_VMReg());
803 reg_def K7_H (SOC, SOC, Op_RegI, 7, k7->as_VMReg()->next());
804
805
806 //----------Architecture Description Register Classes--------------------------
807 // Several register classes are automatically defined based upon information in
808 // this architecture description.
809 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
810 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
811 //
812
813 // Empty register class.
814 reg_class no_reg();
815
816 // Class for all pointer/long registers including APX extended GPRs.
817 reg_class all_reg(RAX, RAX_H,
818 RDX, RDX_H,
819 RBP, RBP_H,
820 RDI, RDI_H,
821 RSI, RSI_H,
822 RCX, RCX_H,
823 RBX, RBX_H,
824 RSP, RSP_H,
825 R8, R8_H,
826 R9, R9_H,
827 R10, R10_H,
828 R11, R11_H,
829 R12, R12_H,
830 R13, R13_H,
831 R14, R14_H,
832 R15, R15_H,
833 R16, R16_H,
834 R17, R17_H,
835 R18, R18_H,
836 R19, R19_H,
837 R20, R20_H,
838 R21, R21_H,
839 R22, R22_H,
840 R23, R23_H,
841 R24, R24_H,
842 R25, R25_H,
843 R26, R26_H,
844 R27, R27_H,
845 R28, R28_H,
846 R29, R29_H,
847 R30, R30_H,
848 R31, R31_H);
849
850 // Class for all int registers including APX extended GPRs.
851 reg_class all_int_reg(RAX
852 RDX,
853 RBP,
854 RDI,
855 RSI,
856 RCX,
857 RBX,
858 R8,
859 R9,
860 R10,
861 R11,
862 R12,
863 R13,
864 R14,
865 R16,
866 R17,
867 R18,
868 R19,
869 R20,
870 R21,
871 R22,
872 R23,
873 R24,
874 R25,
875 R26,
876 R27,
877 R28,
878 R29,
879 R30,
880 R31);
881
882 // Class for all pointer registers
883 reg_class any_reg %{
884 return _ANY_REG_mask;
885 %}
886
887 // Class for all pointer registers (excluding RSP)
888 reg_class ptr_reg %{
889 return _PTR_REG_mask;
890 %}
891
892 // Class for all pointer registers (excluding RSP and RBP)
893 reg_class ptr_reg_no_rbp %{
894 return _PTR_REG_NO_RBP_mask;
895 %}
896
897 // Class for all pointer registers (excluding RAX and RSP)
898 reg_class ptr_no_rax_reg %{
899 return _PTR_NO_RAX_REG_mask;
900 %}
901
902 // Class for all pointer registers (excluding RAX, RBX, and RSP)
903 reg_class ptr_no_rax_rbx_reg %{
904 return _PTR_NO_RAX_RBX_REG_mask;
905 %}
906
907 // Class for all long registers (excluding RSP)
908 reg_class long_reg %{
909 return _LONG_REG_mask;
910 %}
911
912 // Class for all long registers (excluding RAX, RDX and RSP)
913 reg_class long_no_rax_rdx_reg %{
914 return _LONG_NO_RAX_RDX_REG_mask;
915 %}
916
917 // Class for all long registers (excluding RCX and RSP)
918 reg_class long_no_rcx_reg %{
919 return _LONG_NO_RCX_REG_mask;
920 %}
921
922 // Class for all long registers (excluding RBP and R13)
923 reg_class long_no_rbp_r13_reg %{
924 return _LONG_NO_RBP_R13_REG_mask;
925 %}
926
927 // Class for all int registers (excluding RSP)
928 reg_class int_reg %{
929 return _INT_REG_mask;
930 %}
931
932 // Class for all int registers (excluding RAX, RDX, and RSP)
933 reg_class int_no_rax_rdx_reg %{
934 return _INT_NO_RAX_RDX_REG_mask;
935 %}
936
937 // Class for all int registers (excluding RCX and RSP)
938 reg_class int_no_rcx_reg %{
939 return _INT_NO_RCX_REG_mask;
940 %}
941
942 // Class for all int registers (excluding RBP and R13)
943 reg_class int_no_rbp_r13_reg %{
944 return _INT_NO_RBP_R13_REG_mask;
945 %}
946
947 // Singleton class for RAX pointer register
948 reg_class ptr_rax_reg(RAX, RAX_H);
949
950 // Singleton class for RBX pointer register
951 reg_class ptr_rbx_reg(RBX, RBX_H);
952
953 // Singleton class for RSI pointer register
954 reg_class ptr_rsi_reg(RSI, RSI_H);
955
956 // Singleton class for RBP pointer register
957 reg_class ptr_rbp_reg(RBP, RBP_H);
958
959 // Singleton class for RDI pointer register
960 reg_class ptr_rdi_reg(RDI, RDI_H);
961
962 // Singleton class for stack pointer
963 reg_class ptr_rsp_reg(RSP, RSP_H);
964
965 // Singleton class for TLS pointer
966 reg_class ptr_r15_reg(R15, R15_H);
967
968 // Singleton class for RAX long register
969 reg_class long_rax_reg(RAX, RAX_H);
970
971 // Singleton class for RCX long register
972 reg_class long_rcx_reg(RCX, RCX_H);
973
974 // Singleton class for RDX long register
975 reg_class long_rdx_reg(RDX, RDX_H);
976
977 // Singleton class for R11 long register
978 reg_class long_r11_reg(R11, R11_H);
979
980 // Singleton class for RAX int register
981 reg_class int_rax_reg(RAX);
982
983 // Singleton class for RBX int register
984 reg_class int_rbx_reg(RBX);
985
986 // Singleton class for RCX int register
987 reg_class int_rcx_reg(RCX);
988
989 // Singleton class for RDX int register
990 reg_class int_rdx_reg(RDX);
991
992 // Singleton class for RDI int register
993 reg_class int_rdi_reg(RDI);
994
995 // Singleton class for instruction pointer
996 // reg_class ip_reg(RIP);
997
998 alloc_class chunk1(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p,
999 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p,
1000 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p,
1001 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p,
1002 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p,
1003 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p,
1004 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p,
1005 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p,
1006 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p,
1007 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p,
1008 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
1009 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
1010 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
1011 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
1012 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
1013 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p,
1014 XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p,
1015 XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p,
1016 XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p,
1017 XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p,
1018 XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p,
1019 XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p,
1020 XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p,
1021 XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p,
1022 XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p,
1023 XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p,
1024 XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p,
1025 XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p,
1026 XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p,
1027 XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p,
1028 XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p,
1029 XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p);
1030
1031 alloc_class chunk2(K7, K7_H,
1032 K6, K6_H,
1033 K5, K5_H,
1034 K4, K4_H,
1035 K3, K3_H,
1036 K2, K2_H,
1037 K1, K1_H);
1038
1039 reg_class vectmask_reg(K1, K1_H,
1040 K2, K2_H,
1041 K3, K3_H,
1042 K4, K4_H,
1043 K5, K5_H,
1044 K6, K6_H,
1045 K7, K7_H);
1046
1047 reg_class vectmask_reg_K1(K1, K1_H);
1048 reg_class vectmask_reg_K2(K2, K2_H);
1049 reg_class vectmask_reg_K3(K3, K3_H);
1050 reg_class vectmask_reg_K4(K4, K4_H);
1051 reg_class vectmask_reg_K5(K5, K5_H);
1052 reg_class vectmask_reg_K6(K6, K6_H);
1053 reg_class vectmask_reg_K7(K7, K7_H);
1054
1055 // flags allocation class should be last.
1056 alloc_class chunk3(RFLAGS);
1057
1058 // Singleton class for condition codes
1059 reg_class int_flags(RFLAGS);
1060
1061 // Class for pre evex float registers
1062 reg_class float_reg_legacy(XMM0,
1063 XMM1,
1064 XMM2,
1065 XMM3,
1066 XMM4,
1067 XMM5,
1068 XMM6,
1069 XMM7,
1070 XMM8,
1071 XMM9,
1072 XMM10,
1073 XMM11,
1074 XMM12,
1075 XMM13,
1076 XMM14,
1077 XMM15);
1078
1079 // Class for evex float registers
1080 reg_class float_reg_evex(XMM0,
1081 XMM1,
1082 XMM2,
1083 XMM3,
1084 XMM4,
1085 XMM5,
1086 XMM6,
1087 XMM7,
1088 XMM8,
1089 XMM9,
1090 XMM10,
1091 XMM11,
1092 XMM12,
1093 XMM13,
1094 XMM14,
1095 XMM15,
1096 XMM16,
1097 XMM17,
1098 XMM18,
1099 XMM19,
1100 XMM20,
1101 XMM21,
1102 XMM22,
1103 XMM23,
1104 XMM24,
1105 XMM25,
1106 XMM26,
1107 XMM27,
1108 XMM28,
1109 XMM29,
1110 XMM30,
1111 XMM31);
1112
1113 reg_class_dynamic float_reg(float_reg_evex, float_reg_legacy, %{ VM_Version::supports_evex() %} );
1114 reg_class_dynamic float_reg_vl(float_reg_evex, float_reg_legacy, %{ VM_Version::supports_evex() && VM_Version::supports_avx512vl() %} );
1115
1116 // Class for pre evex double registers
1117 reg_class double_reg_legacy(XMM0, XMM0b,
1118 XMM1, XMM1b,
1119 XMM2, XMM2b,
1120 XMM3, XMM3b,
1121 XMM4, XMM4b,
1122 XMM5, XMM5b,
1123 XMM6, XMM6b,
1124 XMM7, XMM7b,
1125 XMM8, XMM8b,
1126 XMM9, XMM9b,
1127 XMM10, XMM10b,
1128 XMM11, XMM11b,
1129 XMM12, XMM12b,
1130 XMM13, XMM13b,
1131 XMM14, XMM14b,
1132 XMM15, XMM15b);
1133
1134 // Class for evex double registers
1135 reg_class double_reg_evex(XMM0, XMM0b,
1136 XMM1, XMM1b,
1137 XMM2, XMM2b,
1138 XMM3, XMM3b,
1139 XMM4, XMM4b,
1140 XMM5, XMM5b,
1141 XMM6, XMM6b,
1142 XMM7, XMM7b,
1143 XMM8, XMM8b,
1144 XMM9, XMM9b,
1145 XMM10, XMM10b,
1146 XMM11, XMM11b,
1147 XMM12, XMM12b,
1148 XMM13, XMM13b,
1149 XMM14, XMM14b,
1150 XMM15, XMM15b,
1151 XMM16, XMM16b,
1152 XMM17, XMM17b,
1153 XMM18, XMM18b,
1154 XMM19, XMM19b,
1155 XMM20, XMM20b,
1156 XMM21, XMM21b,
1157 XMM22, XMM22b,
1158 XMM23, XMM23b,
1159 XMM24, XMM24b,
1160 XMM25, XMM25b,
1161 XMM26, XMM26b,
1162 XMM27, XMM27b,
1163 XMM28, XMM28b,
1164 XMM29, XMM29b,
1165 XMM30, XMM30b,
1166 XMM31, XMM31b);
1167
1168 reg_class_dynamic double_reg(double_reg_evex, double_reg_legacy, %{ VM_Version::supports_evex() %} );
1169 reg_class_dynamic double_reg_vl(double_reg_evex, double_reg_legacy, %{ VM_Version::supports_evex() && VM_Version::supports_avx512vl() %} );
1170
1171 // Class for pre evex 32bit vector registers
1172 reg_class vectors_reg_legacy(XMM0,
1173 XMM1,
1174 XMM2,
1175 XMM3,
1176 XMM4,
1177 XMM5,
1178 XMM6,
1179 XMM7,
1180 XMM8,
1181 XMM9,
1182 XMM10,
1183 XMM11,
1184 XMM12,
1185 XMM13,
1186 XMM14,
1187 XMM15);
1188
1189 // Class for evex 32bit vector registers
1190 reg_class vectors_reg_evex(XMM0,
1191 XMM1,
1192 XMM2,
1193 XMM3,
1194 XMM4,
1195 XMM5,
1196 XMM6,
1197 XMM7,
1198 XMM8,
1199 XMM9,
1200 XMM10,
1201 XMM11,
1202 XMM12,
1203 XMM13,
1204 XMM14,
1205 XMM15,
1206 XMM16,
1207 XMM17,
1208 XMM18,
1209 XMM19,
1210 XMM20,
1211 XMM21,
1212 XMM22,
1213 XMM23,
1214 XMM24,
1215 XMM25,
1216 XMM26,
1217 XMM27,
1218 XMM28,
1219 XMM29,
1220 XMM30,
1221 XMM31);
1222
1223 reg_class_dynamic vectors_reg(vectors_reg_evex, vectors_reg_legacy, %{ VM_Version::supports_evex() %} );
1224 reg_class_dynamic vectors_reg_vlbwdq(vectors_reg_evex, vectors_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1225
1226 // Class for all 64bit vector registers
1227 reg_class vectord_reg_legacy(XMM0, XMM0b,
1228 XMM1, XMM1b,
1229 XMM2, XMM2b,
1230 XMM3, XMM3b,
1231 XMM4, XMM4b,
1232 XMM5, XMM5b,
1233 XMM6, XMM6b,
1234 XMM7, XMM7b,
1235 XMM8, XMM8b,
1236 XMM9, XMM9b,
1237 XMM10, XMM10b,
1238 XMM11, XMM11b,
1239 XMM12, XMM12b,
1240 XMM13, XMM13b,
1241 XMM14, XMM14b,
1242 XMM15, XMM15b);
1243
1244 // Class for all 64bit vector registers
1245 reg_class vectord_reg_evex(XMM0, XMM0b,
1246 XMM1, XMM1b,
1247 XMM2, XMM2b,
1248 XMM3, XMM3b,
1249 XMM4, XMM4b,
1250 XMM5, XMM5b,
1251 XMM6, XMM6b,
1252 XMM7, XMM7b,
1253 XMM8, XMM8b,
1254 XMM9, XMM9b,
1255 XMM10, XMM10b,
1256 XMM11, XMM11b,
1257 XMM12, XMM12b,
1258 XMM13, XMM13b,
1259 XMM14, XMM14b,
1260 XMM15, XMM15b,
1261 XMM16, XMM16b,
1262 XMM17, XMM17b,
1263 XMM18, XMM18b,
1264 XMM19, XMM19b,
1265 XMM20, XMM20b,
1266 XMM21, XMM21b,
1267 XMM22, XMM22b,
1268 XMM23, XMM23b,
1269 XMM24, XMM24b,
1270 XMM25, XMM25b,
1271 XMM26, XMM26b,
1272 XMM27, XMM27b,
1273 XMM28, XMM28b,
1274 XMM29, XMM29b,
1275 XMM30, XMM30b,
1276 XMM31, XMM31b);
1277
1278 reg_class_dynamic vectord_reg(vectord_reg_evex, vectord_reg_legacy, %{ VM_Version::supports_evex() %} );
1279 reg_class_dynamic vectord_reg_vlbwdq(vectord_reg_evex, vectord_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1280
1281 // Class for all 128bit vector registers
1282 reg_class vectorx_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d,
1283 XMM1, XMM1b, XMM1c, XMM1d,
1284 XMM2, XMM2b, XMM2c, XMM2d,
1285 XMM3, XMM3b, XMM3c, XMM3d,
1286 XMM4, XMM4b, XMM4c, XMM4d,
1287 XMM5, XMM5b, XMM5c, XMM5d,
1288 XMM6, XMM6b, XMM6c, XMM6d,
1289 XMM7, XMM7b, XMM7c, XMM7d,
1290 XMM8, XMM8b, XMM8c, XMM8d,
1291 XMM9, XMM9b, XMM9c, XMM9d,
1292 XMM10, XMM10b, XMM10c, XMM10d,
1293 XMM11, XMM11b, XMM11c, XMM11d,
1294 XMM12, XMM12b, XMM12c, XMM12d,
1295 XMM13, XMM13b, XMM13c, XMM13d,
1296 XMM14, XMM14b, XMM14c, XMM14d,
1297 XMM15, XMM15b, XMM15c, XMM15d);
1298
1299 // Class for all 128bit vector registers
1300 reg_class vectorx_reg_evex(XMM0, XMM0b, XMM0c, XMM0d,
1301 XMM1, XMM1b, XMM1c, XMM1d,
1302 XMM2, XMM2b, XMM2c, XMM2d,
1303 XMM3, XMM3b, XMM3c, XMM3d,
1304 XMM4, XMM4b, XMM4c, XMM4d,
1305 XMM5, XMM5b, XMM5c, XMM5d,
1306 XMM6, XMM6b, XMM6c, XMM6d,
1307 XMM7, XMM7b, XMM7c, XMM7d,
1308 XMM8, XMM8b, XMM8c, XMM8d,
1309 XMM9, XMM9b, XMM9c, XMM9d,
1310 XMM10, XMM10b, XMM10c, XMM10d,
1311 XMM11, XMM11b, XMM11c, XMM11d,
1312 XMM12, XMM12b, XMM12c, XMM12d,
1313 XMM13, XMM13b, XMM13c, XMM13d,
1314 XMM14, XMM14b, XMM14c, XMM14d,
1315 XMM15, XMM15b, XMM15c, XMM15d,
1316 XMM16, XMM16b, XMM16c, XMM16d,
1317 XMM17, XMM17b, XMM17c, XMM17d,
1318 XMM18, XMM18b, XMM18c, XMM18d,
1319 XMM19, XMM19b, XMM19c, XMM19d,
1320 XMM20, XMM20b, XMM20c, XMM20d,
1321 XMM21, XMM21b, XMM21c, XMM21d,
1322 XMM22, XMM22b, XMM22c, XMM22d,
1323 XMM23, XMM23b, XMM23c, XMM23d,
1324 XMM24, XMM24b, XMM24c, XMM24d,
1325 XMM25, XMM25b, XMM25c, XMM25d,
1326 XMM26, XMM26b, XMM26c, XMM26d,
1327 XMM27, XMM27b, XMM27c, XMM27d,
1328 XMM28, XMM28b, XMM28c, XMM28d,
1329 XMM29, XMM29b, XMM29c, XMM29d,
1330 XMM30, XMM30b, XMM30c, XMM30d,
1331 XMM31, XMM31b, XMM31c, XMM31d);
1332
1333 reg_class_dynamic vectorx_reg(vectorx_reg_evex, vectorx_reg_legacy, %{ VM_Version::supports_evex() %} );
1334 reg_class_dynamic vectorx_reg_vlbwdq(vectorx_reg_evex, vectorx_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1335
1336 // Class for all 256bit vector registers
1337 reg_class vectory_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h,
1338 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h,
1339 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h,
1340 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h,
1341 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h,
1342 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h,
1343 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h,
1344 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h,
1345 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h,
1346 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h,
1347 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h,
1348 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h,
1349 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h,
1350 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h,
1351 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h,
1352 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h);
1353
1354 // Class for all 256bit vector registers
1355 reg_class vectory_reg_evex(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h,
1356 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h,
1357 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h,
1358 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h,
1359 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h,
1360 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h,
1361 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h,
1362 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h,
1363 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h,
1364 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h,
1365 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h,
1366 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h,
1367 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h,
1368 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h,
1369 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h,
1370 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h,
1371 XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h,
1372 XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h,
1373 XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h,
1374 XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h,
1375 XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h,
1376 XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h,
1377 XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h,
1378 XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h,
1379 XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h,
1380 XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h,
1381 XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h,
1382 XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h,
1383 XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h,
1384 XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h,
1385 XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h,
1386 XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h);
1387
1388 reg_class_dynamic vectory_reg(vectory_reg_evex, vectory_reg_legacy, %{ VM_Version::supports_evex() %} );
1389 reg_class_dynamic vectory_reg_vlbwdq(vectory_reg_evex, vectory_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1390
1391 // Class for all 512bit vector registers
1392 reg_class vectorz_reg_evex(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p,
1393 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p,
1394 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p,
1395 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p,
1396 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p,
1397 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p,
1398 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p,
1399 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p,
1400 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p,
1401 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p,
1402 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
1403 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
1404 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
1405 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
1406 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
1407 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p,
1408 XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p,
1409 XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p,
1410 XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p,
1411 XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p,
1412 XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p,
1413 XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p,
1414 XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p,
1415 XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p,
1416 XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p,
1417 XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p,
1418 XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p,
1419 XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p,
1420 XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p,
1421 XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p,
1422 XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p,
1423 XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p);
1424
1425 // Class for restricted 512bit vector registers
1426 reg_class vectorz_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p,
1427 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p,
1428 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p,
1429 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p,
1430 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p,
1431 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p,
1432 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p,
1433 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p,
1434 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p,
1435 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p,
1436 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
1437 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
1438 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
1439 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
1440 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
1441 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p);
1442
1443 reg_class_dynamic vectorz_reg (vectorz_reg_evex, vectorz_reg_legacy, %{ VM_Version::supports_evex() %} );
1444 reg_class_dynamic vectorz_reg_vl(vectorz_reg_evex, vectorz_reg_legacy, %{ VM_Version::supports_evex() && VM_Version::supports_avx512vl() %} );
1445
1446 reg_class xmm0_reg(XMM0, XMM0b, XMM0c, XMM0d);
1447
1448 %}
1449
1450
1451 //----------SOURCE BLOCK-------------------------------------------------------
1452 // This is a block of C++ code which provides values, functions, and
1453 // definitions necessary in the rest of the architecture description
1454
1455 source_hpp %{
1456
1457 #include "peephole_x86_64.hpp"
1458
1459 bool castLL_is_imm32(const Node* n);
1460
1461 %}
1462
1463 source %{
1464
1465 bool castLL_is_imm32(const Node* n) {
1466 assert(n->is_CastLL(), "must be a CastLL");
1467 const TypeLong* t = n->bottom_type()->is_long();
1468 return (t->_lo == min_jlong || Assembler::is_simm32(t->_lo)) && (t->_hi == max_jlong || Assembler::is_simm32(t->_hi));
1469 }
1470
1471 %}
1472
1473 // Register masks
1474 source_hpp %{
1475
1476 extern RegMask _ANY_REG_mask;
1477 extern RegMask _PTR_REG_mask;
1478 extern RegMask _PTR_REG_NO_RBP_mask;
1479 extern RegMask _PTR_NO_RAX_REG_mask;
1480 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
1481 extern RegMask _LONG_REG_mask;
1482 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
1483 extern RegMask _LONG_NO_RCX_REG_mask;
1484 extern RegMask _LONG_NO_RBP_R13_REG_mask;
1485 extern RegMask _INT_REG_mask;
1486 extern RegMask _INT_NO_RAX_RDX_REG_mask;
1487 extern RegMask _INT_NO_RCX_REG_mask;
1488 extern RegMask _INT_NO_RBP_R13_REG_mask;
1489 extern RegMask _FLOAT_REG_mask;
1490
1491 extern RegMask _STACK_OR_PTR_REG_mask;
1492 extern RegMask _STACK_OR_LONG_REG_mask;
1493 extern RegMask _STACK_OR_INT_REG_mask;
1494
1495 inline const RegMask& STACK_OR_PTR_REG_mask() { return _STACK_OR_PTR_REG_mask; }
1496 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
1497 inline const RegMask& STACK_OR_INT_REG_mask() { return _STACK_OR_INT_REG_mask; }
1498
1499 %}
1500
1501 source %{
1502 #define RELOC_IMM64 Assembler::imm_operand
1503 #define RELOC_DISP32 Assembler::disp32_operand
1504
1505 #define __ masm->
1506
1507 RegMask _ANY_REG_mask;
1508 RegMask _PTR_REG_mask;
1509 RegMask _PTR_REG_NO_RBP_mask;
1510 RegMask _PTR_NO_RAX_REG_mask;
1511 RegMask _PTR_NO_RAX_RBX_REG_mask;
1512 RegMask _LONG_REG_mask;
1513 RegMask _LONG_NO_RAX_RDX_REG_mask;
1514 RegMask _LONG_NO_RCX_REG_mask;
1515 RegMask _LONG_NO_RBP_R13_REG_mask;
1516 RegMask _INT_REG_mask;
1517 RegMask _INT_NO_RAX_RDX_REG_mask;
1518 RegMask _INT_NO_RCX_REG_mask;
1519 RegMask _INT_NO_RBP_R13_REG_mask;
1520 RegMask _FLOAT_REG_mask;
1521 RegMask _STACK_OR_PTR_REG_mask;
1522 RegMask _STACK_OR_LONG_REG_mask;
1523 RegMask _STACK_OR_INT_REG_mask;
1524
1525 static bool need_r12_heapbase() {
1526 return UseCompressedOops;
1527 }
1528
1529 void reg_mask_init() {
1530 constexpr Register egprs[] = {r16, r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31};
1531
1532 // _ALL_REG_mask is generated by adlc from the all_reg register class below.
1533 // We derive a number of subsets from it.
1534 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1535
1536 if (PreserveFramePointer) {
1537 _ANY_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1538 _ANY_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
1539 }
1540 if (need_r12_heapbase()) {
1541 _ANY_REG_mask.remove(OptoReg::as_OptoReg(r12->as_VMReg()));
1542 _ANY_REG_mask.remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
1543 }
1544
1545 _PTR_REG_mask.assignFrom(_ANY_REG_mask);
1546 _PTR_REG_mask.remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
1547 _PTR_REG_mask.remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
1548 _PTR_REG_mask.remove(OptoReg::as_OptoReg(r15->as_VMReg()));
1549 _PTR_REG_mask.remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
1550 if (!UseAPX) {
1551 for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
1552 _PTR_REG_mask.remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
1553 _PTR_REG_mask.remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()->next()));
1554 }
1555 }
1556
1557 _STACK_OR_PTR_REG_mask.assignFrom(_PTR_REG_mask);
1558 _STACK_OR_PTR_REG_mask.or_with(STACK_OR_STACK_SLOTS_mask());
1559
1560 _PTR_REG_NO_RBP_mask.assignFrom(_PTR_REG_mask);
1561 _PTR_REG_NO_RBP_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1562 _PTR_REG_NO_RBP_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
1563
1564 _PTR_NO_RAX_REG_mask.assignFrom(_PTR_REG_mask);
1565 _PTR_NO_RAX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()));
1566 _PTR_NO_RAX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
1567
1568 _PTR_NO_RAX_RBX_REG_mask.assignFrom(_PTR_NO_RAX_REG_mask);
1569 _PTR_NO_RAX_RBX_REG_mask.remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
1570 _PTR_NO_RAX_RBX_REG_mask.remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
1571
1572
1573 _LONG_REG_mask.assignFrom(_PTR_REG_mask);
1574 _STACK_OR_LONG_REG_mask.assignFrom(_LONG_REG_mask);
1575 _STACK_OR_LONG_REG_mask.or_with(STACK_OR_STACK_SLOTS_mask());
1576
1577 _LONG_NO_RAX_RDX_REG_mask.assignFrom(_LONG_REG_mask);
1578 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()));
1579 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
1580 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
1581 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
1582
1583 _LONG_NO_RCX_REG_mask.assignFrom(_LONG_REG_mask);
1584 _LONG_NO_RCX_REG_mask.remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
1585 _LONG_NO_RCX_REG_mask.remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
1586
1587 _LONG_NO_RBP_R13_REG_mask.assignFrom(_LONG_REG_mask);
1588 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1589 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
1590 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(r13->as_VMReg()));
1591 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
1592
1593 _INT_REG_mask.assignFrom(_ALL_INT_REG_mask);
1594 if (!UseAPX) {
1595 for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
1596 _INT_REG_mask.remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
1597 }
1598 }
1599
1600 if (PreserveFramePointer) {
1601 _INT_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1602 }
1603 if (need_r12_heapbase()) {
1604 _INT_REG_mask.remove(OptoReg::as_OptoReg(r12->as_VMReg()));
1605 }
1606
1607 _STACK_OR_INT_REG_mask.assignFrom(_INT_REG_mask);
1608 _STACK_OR_INT_REG_mask.or_with(STACK_OR_STACK_SLOTS_mask());
1609
1610 _INT_NO_RAX_RDX_REG_mask.assignFrom(_INT_REG_mask);
1611 _INT_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()));
1612 _INT_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
1613
1614 _INT_NO_RCX_REG_mask.assignFrom(_INT_REG_mask);
1615 _INT_NO_RCX_REG_mask.remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
1616
1617 _INT_NO_RBP_R13_REG_mask.assignFrom(_INT_REG_mask);
1618 _INT_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1619 _INT_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(r13->as_VMReg()));
1620
1621 // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
1622 // from the float_reg_legacy/float_reg_evex register class.
1623 _FLOAT_REG_mask.assignFrom(VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask);
1624 }
1625
1626 static bool generate_vzeroupper(Compile* C) {
1627 return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false; // Generate vzeroupper
1628 }
1629
1630 static int clear_avx_size() {
1631 return generate_vzeroupper(Compile::current()) ? 3: 0; // vzeroupper
1632 }
1633
1634 // !!!!! Special hack to get all types of calls to specify the byte offset
1635 // from the start of the call to the point where the return address
1636 // will point.
1637 int MachCallStaticJavaNode::ret_addr_offset()
1638 {
1639 int offset = 5; // 5 bytes from start of call to where return address points
1640 offset += clear_avx_size();
1641 return offset;
1642 }
1643
1644 int MachCallDynamicJavaNode::ret_addr_offset()
1645 {
1646 int offset = 15; // 15 bytes from start of call to where return address points
1647 offset += clear_avx_size();
1648 return offset;
1649 }
1650
1651 int MachCallRuntimeNode::ret_addr_offset() {
1652 int offset = 13; // movq r10,#addr; callq (r10)
1653 if (this->ideal_Opcode() != Op_CallLeafVector) {
1654 offset += clear_avx_size();
1655 }
1656 return offset;
1657 }
1658 //
1659 // Compute padding required for nodes which need alignment
1660 //
1661
1662 // The address of the call instruction needs to be 4-byte aligned to
1663 // ensure that it does not span a cache line so that it can be patched.
1664 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
1665 {
1666 current_offset += clear_avx_size(); // skip vzeroupper
1667 current_offset += 1; // skip call opcode byte
1668 return align_up(current_offset, alignment_required()) - current_offset;
1669 }
1670
1671 // The address of the call instruction needs to be 4-byte aligned to
1672 // ensure that it does not span a cache line so that it can be patched.
1673 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
1674 {
1675 current_offset += clear_avx_size(); // skip vzeroupper
1676 current_offset += 11; // skip movq instruction + call opcode byte
1677 return align_up(current_offset, alignment_required()) - current_offset;
1678 }
1679
1680 // This could be in MacroAssembler but it's fairly C2 specific
1681 static void emit_cmpfp_fixup(MacroAssembler* masm) {
1682 Label exit;
1683 __ jccb(Assembler::noParity, exit);
1684 __ pushf();
1685 //
1686 // comiss/ucomiss instructions set ZF,PF,CF flags and
1687 // zero OF,AF,SF for NaN values.
1688 // Fixup flags by zeroing ZF,PF so that compare of NaN
1689 // values returns 'less than' result (CF is set).
1690 // Leave the rest of flags unchanged.
1691 //
1692 // 7 6 5 4 3 2 1 0
1693 // |S|Z|r|A|r|P|r|C| (r - reserved bit)
1694 // 0 0 1 0 1 0 1 1 (0x2B)
1695 //
1696 __ andq(Address(rsp, 0), 0xffffff2b);
1697 __ popf();
1698 __ bind(exit);
1699 }
1700
1701 static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
1702 // If any floating point comparison instruction is used, unordered case always triggers jump
1703 // for below condition, CF=1 is true when at least one input is NaN
1704 Label done;
1705 __ movl(dst, -1);
1706 __ jcc(Assembler::below, done);
1707 __ setcc(Assembler::notEqual, dst);
1708 __ bind(done);
1709 }
1710
1711 enum FP_PREC {
1712 fp_prec_hlf,
1713 fp_prec_flt,
1714 fp_prec_dbl
1715 };
1716
1717 static inline void emit_fp_ucom(MacroAssembler* masm, enum FP_PREC pt,
1718 XMMRegister p, XMMRegister q) {
1719 if (pt == fp_prec_hlf) {
1720 __ evucomish(p, q);
1721 } else if (pt == fp_prec_flt) {
1722 __ ucomiss(p, q);
1723 } else {
1724 __ ucomisd(p, q);
1725 }
1726 }
1727
1728 static inline void movfp(MacroAssembler* masm, enum FP_PREC pt,
1729 XMMRegister dst, XMMRegister src, Register scratch) {
1730 if (pt == fp_prec_hlf) {
1731 __ movhlf(dst, src, scratch);
1732 } else if (pt == fp_prec_flt) {
1733 __ movflt(dst, src);
1734 } else {
1735 __ movdbl(dst, src);
1736 }
1737 }
1738
1739 // Math.min() # Math.max()
1740 // -----------------------------
1741 // (v)ucomis[h/s/d] #
1742 // ja -> b # a
1743 // jp -> NaN # NaN
1744 // jb -> a # b
1745 // je -> a | b # a & b
1746 static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
1747 XMMRegister a, XMMRegister b, Register rt,
1748 bool min, enum FP_PREC pt) {
1749 Label nan, zero, below, above, done;
1750
1751 emit_fp_ucom(masm, pt, a, b);
1752
1753 if (dst->encoding() != (min ? b : a)->encoding()) {
1754 __ jccb(Assembler::above, above); // CF=0 & ZF=0
1755 } else {
1756 __ jccb(Assembler::above, done);
1757 }
1758 __ jccb(Assembler::parity, nan); // PF=1
1759 __ jccb(Assembler::below, below); // CF=1
1760
1761 // equal
1762 // Using bitwise operations is a low cost way to compute the correct result
1763 // for zero and non-zero inputs in this scenario except for NaN, which is
1764 // handled separately. The mantissa and exponent are valid with either
1765 // bitwise operation. For zero inputs, the sign bit is chosen according to
1766 // whether a minimum or maximum value is required.
1767 if (min) {
1768 // Negative sign preserved when available (e.g., min(+0, -0) -> -0)
1769 __ vpor(dst, a, b, Assembler::AVX_128bit);
1770 } else {
1771 // Positive sign preserved when available (e.g., max(+0, -0) -> +0)
1772 __ vpand(dst, a, b, Assembler::AVX_128bit);
1773 }
1774 __ jmp(done);
1775
1776 __ bind(above);
1777 movfp(masm, pt, dst, min ? b : a, rt);
1778 __ jmp(done);
1779
1780 __ bind(nan);
1781 if (pt == fp_prec_hlf) {
1782 __ movl(rt, 0x00007e00); // Float16.NaN
1783 __ evmovw(dst, rt);
1784 } else if (pt == fp_prec_flt) {
1785 __ movl(rt, 0x7fc00000); // Float.NaN
1786 __ movdl(dst, rt);
1787 } else {
1788 __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
1789 __ movdq(dst, rt);
1790 }
1791 __ jmp(done);
1792
1793 __ bind(below);
1794 movfp(masm, pt, dst, min ? a : b, rt);
1795
1796 __ bind(done);
1797 }
1798
1799 //=============================================================================
1800 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1801
1802 int ConstantTable::calculate_table_base_offset() const {
1803 return 0; // absolute addressing, no offset
1804 }
1805
1806 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1807 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1808 ShouldNotReachHere();
1809 }
1810
1811 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1812 // Empty encoding
1813 }
1814
1815 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1816 return 0;
1817 }
1818
1819 #ifndef PRODUCT
1820 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1821 st->print("# MachConstantBaseNode (empty encoding)");
1822 }
1823 #endif
1824
1825
1826 //=============================================================================
1827 #ifndef PRODUCT
1828 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1829 Compile* C = ra_->C;
1830
1831 int framesize = C->output()->frame_size_in_bytes();
1832 int bangsize = C->output()->bang_size_in_bytes();
1833 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1834 // Remove wordSize for return addr which is already pushed.
1835 framesize -= wordSize;
1836
1837 if (C->output()->need_stack_bang(bangsize)) {
1838 framesize -= wordSize;
1839 st->print("# stack bang (%d bytes)", bangsize);
1840 st->print("\n\t");
1841 st->print("pushq rbp\t# Save rbp");
1842 if (PreserveFramePointer) {
1843 st->print("\n\t");
1844 st->print("movq rbp, rsp\t# Save the caller's SP into rbp");
1845 }
1846 if (framesize) {
1847 st->print("\n\t");
1848 st->print("subq rsp, #%d\t# Create frame",framesize);
1849 }
1850 } else {
1851 st->print("subq rsp, #%d\t# Create frame",framesize);
1852 st->print("\n\t");
1853 framesize -= wordSize;
1854 st->print("movq [rsp + #%d], rbp\t# Save rbp",framesize);
1855 if (PreserveFramePointer) {
1856 st->print("\n\t");
1857 st->print("movq rbp, rsp\t# Save the caller's SP into rbp");
1858 if (framesize > 0) {
1859 st->print("\n\t");
1860 st->print("addq rbp, #%d", framesize);
1861 }
1862 }
1863 }
1864
1865 if (VerifyStackAtCalls) {
1866 st->print("\n\t");
1867 framesize -= wordSize;
1868 st->print("movq [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
1869 #ifdef ASSERT
1870 st->print("\n\t");
1871 st->print("# stack alignment check");
1872 #endif
1873 }
1874 if (C->stub_function() != nullptr) {
1875 st->print("\n\t");
1876 st->print("cmpl [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
1877 st->print("\n\t");
1878 st->print("je fast_entry\t");
1879 st->print("\n\t");
1880 st->print("call #nmethod_entry_barrier_stub\t");
1881 st->print("\n\tfast_entry:");
1882 }
1883 st->cr();
1884 }
1885 #endif
1886
1887 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1888 Compile* C = ra_->C;
1889
1890 int framesize = C->output()->frame_size_in_bytes();
1891 int bangsize = C->output()->bang_size_in_bytes();
1892
1893 if (C->clinit_barrier_on_entry()) {
1894 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
1895 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1896
1897 Label L_skip_barrier;
1898 Register klass = rscratch1;
1899
1900 __ mov_metadata(klass, C->method()->holder()->constant_encoding());
1901 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
1902
1903 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
1904
1905 __ bind(L_skip_barrier);
1906 }
1907
1908 __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
1909
1910 C->output()->set_frame_complete(__ offset());
1911
1912 if (C->has_mach_constant_base_node()) {
1913 // NOTE: We set the table base offset here because users might be
1914 // emitted before MachConstantBaseNode.
1915 ConstantTable& constant_table = C->output()->constant_table();
1916 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1917 }
1918 }
1919
1920 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1921 {
1922 return MachNode::size(ra_); // too many variables; just compute it
1923 // the hard way
1924 }
1925
1926 int MachPrologNode::reloc() const
1927 {
1928 return 0; // a large enough number
1929 }
1930
1931 //=============================================================================
1932 #ifndef PRODUCT
1933 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1934 {
1935 Compile* C = ra_->C;
1936 if (generate_vzeroupper(C)) {
1937 st->print("vzeroupper");
1938 st->cr(); st->print("\t");
1939 }
1940
1941 int framesize = C->output()->frame_size_in_bytes();
1942 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1943 // Remove word for return adr already pushed
1944 // and RBP
1945 framesize -= 2*wordSize;
1946
1947 if (framesize) {
1948 st->print_cr("addq rsp, %d\t# Destroy frame", framesize);
1949 st->print("\t");
1950 }
1951
1952 st->print_cr("popq rbp");
1953 if (do_polling() && C->is_method_compilation()) {
1954 st->print("\t");
1955 st->print_cr("cmpq rsp, poll_offset[r15_thread] \n\t"
1956 "ja #safepoint_stub\t"
1957 "# Safepoint: poll for GC");
1958 }
1959 }
1960 #endif
1961
1962 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
1963 {
1964 Compile* C = ra_->C;
1965
1966 if (generate_vzeroupper(C)) {
1967 // Clear upper bits of YMM registers when current compiled code uses
1968 // wide vectors to avoid AVX <-> SSE transition penalty during call.
1969 __ vzeroupper();
1970 }
1971
1972 int framesize = C->output()->frame_size_in_bytes();
1973 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1974 // Remove word for return adr already pushed
1975 // and RBP
1976 framesize -= 2*wordSize;
1977
1978 // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
1979
1980 if (framesize) {
1981 __ addq(rsp, framesize);
1982 }
1983
1984 __ popq(rbp);
1985
1986 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1987 __ reserved_stack_check();
1988 }
1989
1990 if (do_polling() && C->is_method_compilation()) {
1991 Label dummy_label;
1992 Label* code_stub = &dummy_label;
1993 if (!C->output()->in_scratch_emit_size()) {
1994 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1995 C->output()->add_stub(stub);
1996 code_stub = &stub->entry();
1997 }
1998 __ relocate(relocInfo::poll_return_type);
1999 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
2000 }
2001 }
2002
2003 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
2004 {
2005 return MachNode::size(ra_); // too many variables; just compute it
2006 // the hard way
2007 }
2008
2009 int MachEpilogNode::reloc() const
2010 {
2011 return 2; // a large enough number
2012 }
2013
2014 const Pipeline* MachEpilogNode::pipeline() const
2015 {
2016 return MachNode::pipeline_class();
2017 }
2018
2019 //=============================================================================
2020
2021 enum RC {
2022 rc_bad,
2023 rc_int,
2024 rc_kreg,
2025 rc_float,
2026 rc_stack
2027 };
2028
2029 static enum RC rc_class(OptoReg::Name reg)
2030 {
2031 if( !OptoReg::is_valid(reg) ) return rc_bad;
2032
2033 if (OptoReg::is_stack(reg)) return rc_stack;
2034
2035 VMReg r = OptoReg::as_VMReg(reg);
2036
2037 if (r->is_Register()) return rc_int;
2038
2039 if (r->is_KRegister()) return rc_kreg;
2040
2041 assert(r->is_XMMRegister(), "must be");
2042 return rc_float;
2043 }
2044
2045 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
2046 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
2047 int src_hi, int dst_hi, uint ireg, outputStream* st);
2048
2049 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
2050 int stack_offset, int reg, uint ireg, outputStream* st);
2051
2052 static void vec_stack_to_stack_helper(C2_MacroAssembler *masm, int src_offset,
2053 int dst_offset, uint ireg, outputStream* st) {
2054 if (masm) {
2055 switch (ireg) {
2056 case Op_VecS:
2057 __ movq(Address(rsp, -8), rax);
2058 __ movl(rax, Address(rsp, src_offset));
2059 __ movl(Address(rsp, dst_offset), rax);
2060 __ movq(rax, Address(rsp, -8));
2061 break;
2062 case Op_VecD:
2063 __ pushq(Address(rsp, src_offset));
2064 __ popq (Address(rsp, dst_offset));
2065 break;
2066 case Op_VecX:
2067 __ pushq(Address(rsp, src_offset));
2068 __ popq (Address(rsp, dst_offset));
2069 __ pushq(Address(rsp, src_offset+8));
2070 __ popq (Address(rsp, dst_offset+8));
2071 break;
2072 case Op_VecY:
2073 __ vmovdqu(Address(rsp, -32), xmm0);
2074 __ vmovdqu(xmm0, Address(rsp, src_offset));
2075 __ vmovdqu(Address(rsp, dst_offset), xmm0);
2076 __ vmovdqu(xmm0, Address(rsp, -32));
2077 break;
2078 case Op_VecZ:
2079 __ evmovdquq(Address(rsp, -64), xmm0, 2);
2080 __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
2081 __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
2082 __ evmovdquq(xmm0, Address(rsp, -64), 2);
2083 break;
2084 default:
2085 ShouldNotReachHere();
2086 }
2087 #ifndef PRODUCT
2088 } else {
2089 switch (ireg) {
2090 case Op_VecS:
2091 st->print("movq [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
2092 "movl rax, [rsp + #%d]\n\t"
2093 "movl [rsp + #%d], rax\n\t"
2094 "movq rax, [rsp - #8]",
2095 src_offset, dst_offset);
2096 break;
2097 case Op_VecD:
2098 st->print("pushq [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
2099 "popq [rsp + #%d]",
2100 src_offset, dst_offset);
2101 break;
2102 case Op_VecX:
2103 st->print("pushq [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
2104 "popq [rsp + #%d]\n\t"
2105 "pushq [rsp + #%d]\n\t"
2106 "popq [rsp + #%d]",
2107 src_offset, dst_offset, src_offset+8, dst_offset+8);
2108 break;
2109 case Op_VecY:
2110 st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
2111 "vmovdqu xmm0, [rsp + #%d]\n\t"
2112 "vmovdqu [rsp + #%d], xmm0\n\t"
2113 "vmovdqu xmm0, [rsp - #32]",
2114 src_offset, dst_offset);
2115 break;
2116 case Op_VecZ:
2117 st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
2118 "vmovdqu xmm0, [rsp + #%d]\n\t"
2119 "vmovdqu [rsp + #%d], xmm0\n\t"
2120 "vmovdqu xmm0, [rsp - #64]",
2121 src_offset, dst_offset);
2122 break;
2123 default:
2124 ShouldNotReachHere();
2125 }
2126 #endif
2127 }
2128 }
2129
2130 uint MachSpillCopyNode::implementation(C2_MacroAssembler* masm,
2131 PhaseRegAlloc* ra_,
2132 bool do_size,
2133 outputStream* st) const {
2134 assert(masm != nullptr || st != nullptr, "sanity");
2135 // Get registers to move
2136 OptoReg::Name src_second = ra_->get_reg_second(in(1));
2137 OptoReg::Name src_first = ra_->get_reg_first(in(1));
2138 OptoReg::Name dst_second = ra_->get_reg_second(this);
2139 OptoReg::Name dst_first = ra_->get_reg_first(this);
2140
2141 enum RC src_second_rc = rc_class(src_second);
2142 enum RC src_first_rc = rc_class(src_first);
2143 enum RC dst_second_rc = rc_class(dst_second);
2144 enum RC dst_first_rc = rc_class(dst_first);
2145
2146 assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
2147 "must move at least 1 register" );
2148
2149 if (src_first == dst_first && src_second == dst_second) {
2150 // Self copy, no move
2151 return 0;
2152 }
2153 if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_pvectmask() == nullptr) {
2154 uint ireg = ideal_reg();
2155 assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
2156 assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
2157 if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
2158 // mem -> mem
2159 int src_offset = ra_->reg2offset(src_first);
2160 int dst_offset = ra_->reg2offset(dst_first);
2161 vec_stack_to_stack_helper(masm, src_offset, dst_offset, ireg, st);
2162 } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
2163 vec_mov_helper(masm, src_first, dst_first, src_second, dst_second, ireg, st);
2164 } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
2165 int stack_offset = ra_->reg2offset(dst_first);
2166 vec_spill_helper(masm, false, stack_offset, src_first, ireg, st);
2167 } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
2168 int stack_offset = ra_->reg2offset(src_first);
2169 vec_spill_helper(masm, true, stack_offset, dst_first, ireg, st);
2170 } else {
2171 ShouldNotReachHere();
2172 }
2173 return 0;
2174 }
2175 if (src_first_rc == rc_stack) {
2176 // mem ->
2177 if (dst_first_rc == rc_stack) {
2178 // mem -> mem
2179 assert(src_second != dst_first, "overlap");
2180 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2181 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2182 // 64-bit
2183 int src_offset = ra_->reg2offset(src_first);
2184 int dst_offset = ra_->reg2offset(dst_first);
2185 if (masm) {
2186 __ pushq(Address(rsp, src_offset));
2187 __ popq (Address(rsp, dst_offset));
2188 #ifndef PRODUCT
2189 } else {
2190 st->print("pushq [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
2191 "popq [rsp + #%d]",
2192 src_offset, dst_offset);
2193 #endif
2194 }
2195 } else {
2196 // 32-bit
2197 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2198 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2199 // No pushl/popl, so:
2200 int src_offset = ra_->reg2offset(src_first);
2201 int dst_offset = ra_->reg2offset(dst_first);
2202 if (masm) {
2203 __ movq(Address(rsp, -8), rax);
2204 __ movl(rax, Address(rsp, src_offset));
2205 __ movl(Address(rsp, dst_offset), rax);
2206 __ movq(rax, Address(rsp, -8));
2207 #ifndef PRODUCT
2208 } else {
2209 st->print("movq [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
2210 "movl rax, [rsp + #%d]\n\t"
2211 "movl [rsp + #%d], rax\n\t"
2212 "movq rax, [rsp - #8]",
2213 src_offset, dst_offset);
2214 #endif
2215 }
2216 }
2217 return 0;
2218 } else if (dst_first_rc == rc_int) {
2219 // mem -> gpr
2220 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2221 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2222 // 64-bit
2223 int offset = ra_->reg2offset(src_first);
2224 if (masm) {
2225 __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2226 #ifndef PRODUCT
2227 } else {
2228 st->print("movq %s, [rsp + #%d]\t# spill",
2229 Matcher::regName[dst_first],
2230 offset);
2231 #endif
2232 }
2233 } else {
2234 // 32-bit
2235 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2236 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2237 int offset = ra_->reg2offset(src_first);
2238 if (masm) {
2239 __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2240 #ifndef PRODUCT
2241 } else {
2242 st->print("movl %s, [rsp + #%d]\t# spill",
2243 Matcher::regName[dst_first],
2244 offset);
2245 #endif
2246 }
2247 }
2248 return 0;
2249 } else if (dst_first_rc == rc_float) {
2250 // mem-> xmm
2251 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2252 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2253 // 64-bit
2254 int offset = ra_->reg2offset(src_first);
2255 if (masm) {
2256 __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2257 #ifndef PRODUCT
2258 } else {
2259 st->print("%s %s, [rsp + #%d]\t# spill",
2260 UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
2261 Matcher::regName[dst_first],
2262 offset);
2263 #endif
2264 }
2265 } else {
2266 // 32-bit
2267 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2268 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2269 int offset = ra_->reg2offset(src_first);
2270 if (masm) {
2271 __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2272 #ifndef PRODUCT
2273 } else {
2274 st->print("movss %s, [rsp + #%d]\t# spill",
2275 Matcher::regName[dst_first],
2276 offset);
2277 #endif
2278 }
2279 }
2280 return 0;
2281 } else if (dst_first_rc == rc_kreg) {
2282 // mem -> kreg
2283 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2284 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2285 // 64-bit
2286 int offset = ra_->reg2offset(src_first);
2287 if (masm) {
2288 __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2289 #ifndef PRODUCT
2290 } else {
2291 st->print("kmovq %s, [rsp + #%d]\t# spill",
2292 Matcher::regName[dst_first],
2293 offset);
2294 #endif
2295 }
2296 }
2297 return 0;
2298 }
2299 } else if (src_first_rc == rc_int) {
2300 // gpr ->
2301 if (dst_first_rc == rc_stack) {
2302 // gpr -> mem
2303 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2304 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2305 // 64-bit
2306 int offset = ra_->reg2offset(dst_first);
2307 if (masm) {
2308 __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
2309 #ifndef PRODUCT
2310 } else {
2311 st->print("movq [rsp + #%d], %s\t# spill",
2312 offset,
2313 Matcher::regName[src_first]);
2314 #endif
2315 }
2316 } else {
2317 // 32-bit
2318 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2319 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2320 int offset = ra_->reg2offset(dst_first);
2321 if (masm) {
2322 __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
2323 #ifndef PRODUCT
2324 } else {
2325 st->print("movl [rsp + #%d], %s\t# spill",
2326 offset,
2327 Matcher::regName[src_first]);
2328 #endif
2329 }
2330 }
2331 return 0;
2332 } else if (dst_first_rc == rc_int) {
2333 // gpr -> gpr
2334 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2335 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2336 // 64-bit
2337 if (masm) {
2338 __ movq(as_Register(Matcher::_regEncode[dst_first]),
2339 as_Register(Matcher::_regEncode[src_first]));
2340 #ifndef PRODUCT
2341 } else {
2342 st->print("movq %s, %s\t# spill",
2343 Matcher::regName[dst_first],
2344 Matcher::regName[src_first]);
2345 #endif
2346 }
2347 return 0;
2348 } else {
2349 // 32-bit
2350 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2351 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2352 if (masm) {
2353 __ movl(as_Register(Matcher::_regEncode[dst_first]),
2354 as_Register(Matcher::_regEncode[src_first]));
2355 #ifndef PRODUCT
2356 } else {
2357 st->print("movl %s, %s\t# spill",
2358 Matcher::regName[dst_first],
2359 Matcher::regName[src_first]);
2360 #endif
2361 }
2362 return 0;
2363 }
2364 } else if (dst_first_rc == rc_float) {
2365 // gpr -> xmm
2366 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2367 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2368 // 64-bit
2369 if (masm) {
2370 __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
2371 #ifndef PRODUCT
2372 } else {
2373 st->print("movdq %s, %s\t# spill",
2374 Matcher::regName[dst_first],
2375 Matcher::regName[src_first]);
2376 #endif
2377 }
2378 } else {
2379 // 32-bit
2380 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2381 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2382 if (masm) {
2383 __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
2384 #ifndef PRODUCT
2385 } else {
2386 st->print("movdl %s, %s\t# spill",
2387 Matcher::regName[dst_first],
2388 Matcher::regName[src_first]);
2389 #endif
2390 }
2391 }
2392 return 0;
2393 } else if (dst_first_rc == rc_kreg) {
2394 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2395 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2396 // 64-bit
2397 if (masm) {
2398 __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
2399 #ifndef PRODUCT
2400 } else {
2401 st->print("kmovq %s, %s\t# spill",
2402 Matcher::regName[dst_first],
2403 Matcher::regName[src_first]);
2404 #endif
2405 }
2406 }
2407 Unimplemented();
2408 return 0;
2409 }
2410 } else if (src_first_rc == rc_float) {
2411 // xmm ->
2412 if (dst_first_rc == rc_stack) {
2413 // xmm -> mem
2414 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2415 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2416 // 64-bit
2417 int offset = ra_->reg2offset(dst_first);
2418 if (masm) {
2419 __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
2420 #ifndef PRODUCT
2421 } else {
2422 st->print("movsd [rsp + #%d], %s\t# spill",
2423 offset,
2424 Matcher::regName[src_first]);
2425 #endif
2426 }
2427 } else {
2428 // 32-bit
2429 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2430 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2431 int offset = ra_->reg2offset(dst_first);
2432 if (masm) {
2433 __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
2434 #ifndef PRODUCT
2435 } else {
2436 st->print("movss [rsp + #%d], %s\t# spill",
2437 offset,
2438 Matcher::regName[src_first]);
2439 #endif
2440 }
2441 }
2442 return 0;
2443 } else if (dst_first_rc == rc_int) {
2444 // xmm -> gpr
2445 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2446 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2447 // 64-bit
2448 if (masm) {
2449 __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2450 #ifndef PRODUCT
2451 } else {
2452 st->print("movdq %s, %s\t# spill",
2453 Matcher::regName[dst_first],
2454 Matcher::regName[src_first]);
2455 #endif
2456 }
2457 } else {
2458 // 32-bit
2459 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2460 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2461 if (masm) {
2462 __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2463 #ifndef PRODUCT
2464 } else {
2465 st->print("movdl %s, %s\t# spill",
2466 Matcher::regName[dst_first],
2467 Matcher::regName[src_first]);
2468 #endif
2469 }
2470 }
2471 return 0;
2472 } else if (dst_first_rc == rc_float) {
2473 // xmm -> xmm
2474 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2475 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2476 // 64-bit
2477 if (masm) {
2478 __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2479 #ifndef PRODUCT
2480 } else {
2481 st->print("%s %s, %s\t# spill",
2482 UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
2483 Matcher::regName[dst_first],
2484 Matcher::regName[src_first]);
2485 #endif
2486 }
2487 } else {
2488 // 32-bit
2489 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2490 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2491 if (masm) {
2492 __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2493 #ifndef PRODUCT
2494 } else {
2495 st->print("%s %s, %s\t# spill",
2496 UseXmmRegToRegMoveAll ? "movaps" : "movss ",
2497 Matcher::regName[dst_first],
2498 Matcher::regName[src_first]);
2499 #endif
2500 }
2501 }
2502 return 0;
2503 } else if (dst_first_rc == rc_kreg) {
2504 assert(false, "Illegal spilling");
2505 return 0;
2506 }
2507 } else if (src_first_rc == rc_kreg) {
2508 if (dst_first_rc == rc_stack) {
2509 // mem -> kreg
2510 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2511 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2512 // 64-bit
2513 int offset = ra_->reg2offset(dst_first);
2514 if (masm) {
2515 __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
2516 #ifndef PRODUCT
2517 } else {
2518 st->print("kmovq [rsp + #%d] , %s\t# spill",
2519 offset,
2520 Matcher::regName[src_first]);
2521 #endif
2522 }
2523 }
2524 return 0;
2525 } else if (dst_first_rc == rc_int) {
2526 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2527 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2528 // 64-bit
2529 if (masm) {
2530 __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
2531 #ifndef PRODUCT
2532 } else {
2533 st->print("kmovq %s, %s\t# spill",
2534 Matcher::regName[dst_first],
2535 Matcher::regName[src_first]);
2536 #endif
2537 }
2538 }
2539 Unimplemented();
2540 return 0;
2541 } else if (dst_first_rc == rc_kreg) {
2542 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2543 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2544 // 64-bit
2545 if (masm) {
2546 __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
2547 #ifndef PRODUCT
2548 } else {
2549 st->print("kmovq %s, %s\t# spill",
2550 Matcher::regName[dst_first],
2551 Matcher::regName[src_first]);
2552 #endif
2553 }
2554 }
2555 return 0;
2556 } else if (dst_first_rc == rc_float) {
2557 assert(false, "Illegal spill");
2558 return 0;
2559 }
2560 }
2561
2562 assert(0," foo ");
2563 Unimplemented();
2564 return 0;
2565 }
2566
2567 #ifndef PRODUCT
2568 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
2569 implementation(nullptr, ra_, false, st);
2570 }
2571 #endif
2572
2573 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2574 implementation(masm, ra_, false, nullptr);
2575 }
2576
2577 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2578 return MachNode::size(ra_);
2579 }
2580
2581 //=============================================================================
2582 #ifndef PRODUCT
2583 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2584 {
2585 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2586 int reg = ra_->get_reg_first(this);
2587 st->print("leaq %s, [rsp + #%d]\t# box lock",
2588 Matcher::regName[reg], offset);
2589 }
2590 #endif
2591
2592 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2593 {
2594 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2595 int reg = ra_->get_encode(this);
2596
2597 __ lea(as_Register(reg), Address(rsp, offset));
2598 }
2599
2600 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
2601 {
2602 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2603 if (ra_->get_encode(this) > 15) {
2604 return (offset < 0x80) ? 6 : 9; // REX2
2605 } else {
2606 return (offset < 0x80) ? 5 : 8; // REX
2607 }
2608 }
2609
2610 //=============================================================================
2611 #ifndef PRODUCT
2612 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2613 {
2614 st->print_cr("movl rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2615 st->print_cr("\tcmpl rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
2616 st->print_cr("\tjne SharedRuntime::_ic_miss_stub");
2617 }
2618 #endif
2619
2620 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2621 {
2622 __ ic_check(InteriorEntryAlignment);
2623 }
2624
2625 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2626 {
2627 return MachNode::size(ra_); // too many variables; just compute it
2628 // the hard way
2629 }
2630
2631
2632 //=============================================================================
2633
2634 bool Matcher::supports_vector_calling_convention(void) {
2635 return EnableVectorSupport;
2636 }
2637
2638 static bool is_ndd_demotable_opr1(const MachNode* mdef) {
2639 return ((mdef->flags() & Node::PD::Flag_ndd_demotable_opr1) != 0);
2640 }
2641
2642 static bool is_ndd_demotable_opr2(const MachNode* mdef) {
2643 return ((mdef->flags() & Node::PD::Flag_ndd_demotable_opr2) != 0);
2644 }
2645
2646 #ifdef ASSERT
2647 static bool is_ndd_demotable(const MachNode* mdef) {
2648 return (is_ndd_demotable_opr1(mdef) || is_ndd_demotable_opr2(mdef));
2649 }
2650 #endif
2651
2652 bool Matcher::is_register_biasing_candidate(const MachNode* mdef,
2653 int oper_index) {
2654 if (mdef == nullptr) {
2655 return false;
2656 }
2657
2658 if (mdef->num_opnds() <= oper_index || mdef->operand_index(oper_index) < 0 ||
2659 mdef->in(mdef->operand_index(oper_index)) == nullptr) {
2660 assert(oper_index != 1 || !is_ndd_demotable_opr1(mdef), "%s", mdef->Name());
2661 assert(oper_index != 2 || !is_ndd_demotable_opr2(mdef), "%s", mdef->Name());
2662 return false;
2663 }
2664
2665 // Complex memory operand covers multiple incoming edges needed for
2666 // address computation. Biasing def towards any address component will not
2667 // result in NDD demotion by assembler.
2668 if (mdef->operand_num_edges(oper_index) != 1) {
2669 return false;
2670 }
2671
2672 // Demotion candidate must be register mask compatible with definition.
2673 const RegMask& oper_mask = mdef->in_RegMask(mdef->operand_index(oper_index));
2674 if (!oper_mask.overlap(mdef->out_RegMask())) {
2675 assert(!is_ndd_demotable(mdef), "%s", mdef->Name());
2676 return false;
2677 }
2678
2679 switch (oper_index) {
2680 // First operand of MachNode corresponding to Intel APX NDD selection
2681 // pattern can share its assigned register with definition operand if
2682 // their live ranges do not overlap. In such a scenario we can demote
2683 // it to legacy map0/map1 instruction by replacing its 4-byte extended
2684 // EVEX prefix with shorter REX/REX2 encoding. Demotion candidates
2685 // are decorated with a special flag by instruction selector.
2686 case 1:
2687 return is_ndd_demotable_opr1(mdef);
2688
2689 // Definition operand of commutative operation can be biased towards second
2690 // operand.
2691 case 2:
2692 return is_ndd_demotable_opr2(mdef);
2693
2694 // Current scheme only selects up to two biasing candidates
2695 default:
2696 assert(false, "unhandled operand index: %s", mdef->Name());
2697 break;
2698 }
2699
2700 return false;
2701 }
2702
2703 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2704 assert(EnableVectorSupport, "sanity");
2705 int lo = XMM0_num;
2706 int hi = XMM0b_num;
2707 if (ideal_reg == Op_VecX) hi = XMM0d_num;
2708 else if (ideal_reg == Op_VecY) hi = XMM0h_num;
2709 else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
2710 return OptoRegPair(hi, lo);
2711 }
2712
2713 // Is this branch offset short enough that a short branch can be used?
2714 //
2715 // NOTE: If the platform does not provide any short branch variants, then
2716 // this method should return false for offset 0.
2717 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2718 // The passed offset is relative to address of the branch.
2719 // On 86 a branch displacement is calculated relative to address
2720 // of a next instruction.
2721 offset -= br_size;
2722
2723 // the short version of jmpConUCF2 contains multiple branches,
2724 // making the reach slightly less
2725 if (rule == jmpConUCF2_rule)
2726 return (-126 <= offset && offset <= 125);
2727 return (-128 <= offset && offset <= 127);
2728 }
2729
2730 #ifdef ASSERT
2731 // Return whether or not this register is ever used as an argument.
2732 bool Matcher::can_be_java_arg(int reg)
2733 {
2734 return
2735 reg == RDI_num || reg == RDI_H_num ||
2736 reg == RSI_num || reg == RSI_H_num ||
2737 reg == RDX_num || reg == RDX_H_num ||
2738 reg == RCX_num || reg == RCX_H_num ||
2739 reg == R8_num || reg == R8_H_num ||
2740 reg == R9_num || reg == R9_H_num ||
2741 reg == R12_num || reg == R12_H_num ||
2742 reg == XMM0_num || reg == XMM0b_num ||
2743 reg == XMM1_num || reg == XMM1b_num ||
2744 reg == XMM2_num || reg == XMM2b_num ||
2745 reg == XMM3_num || reg == XMM3b_num ||
2746 reg == XMM4_num || reg == XMM4b_num ||
2747 reg == XMM5_num || reg == XMM5b_num ||
2748 reg == XMM6_num || reg == XMM6b_num ||
2749 reg == XMM7_num || reg == XMM7b_num;
2750 }
2751 #endif
2752
2753 uint Matcher::int_pressure_limit()
2754 {
2755 return (INTPRESSURE == -1) ? _INT_REG_mask.size() : INTPRESSURE;
2756 }
2757
2758 uint Matcher::float_pressure_limit()
2759 {
2760 // After experiment around with different values, the following default threshold
2761 // works best for LCM's register pressure scheduling on x64.
2762 uint dec_count = VM_Version::supports_evex() ? 4 : 2;
2763 uint default_float_pressure_threshold = _FLOAT_REG_mask.size() - dec_count;
2764 return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
2765 }
2766
2767 // Register for the first projection of an int pair
2768 const RegMask& Matcher::firstI_proj_mask() {
2769 return INT_RAX_REG_mask();
2770 }
2771
2772 // Register for the second projection of an int pair
2773 const RegMask& Matcher::secondI_proj_mask() {
2774 return INT_RDX_REG_mask();
2775 }
2776
2777 // Register for the first projection of a long pair
2778 const RegMask& Matcher::firstL_proj_mask() {
2779 return LONG_RAX_REG_mask();
2780 }
2781
2782 // Register for the second projection of a long pair
2783 const RegMask& Matcher::secondL_proj_mask() {
2784 return LONG_RDX_REG_mask();
2785 }
2786
2787 %}
2788
2789 source_hpp %{
2790 // Header information of the source block.
2791 // Method declarations/definitions which are used outside
2792 // the ad-scope can conveniently be defined here.
2793 //
2794 // To keep related declarations/definitions/uses close together,
2795 // we switch between source %{ }% and source_hpp %{ }% freely as needed.
2796
2797 #include "runtime/vm_version.hpp"
2798
2799 class NativeJump;
2800
2801 class CallStubImpl {
2802
2803 //--------------------------------------------------------------
2804 //---< Used for optimization in Compile::shorten_branches >---
2805 //--------------------------------------------------------------
2806
2807 public:
2808 // Size of call trampoline stub.
2809 static uint size_call_trampoline() {
2810 return 0; // no call trampolines on this platform
2811 }
2812
2813 // number of relocations needed by a call trampoline stub
2814 static uint reloc_call_trampoline() {
2815 return 0; // no call trampolines on this platform
2816 }
2817 };
2818
2819 class HandlerImpl {
2820
2821 public:
2822
2823 static int emit_deopt_handler(C2_MacroAssembler* masm);
2824
2825 static uint size_deopt_handler() {
2826 // one call and one jmp.
2827 return 7;
2828 }
2829 };
2830
2831 inline Assembler::AvxVectorLen vector_length_encoding(int bytes) {
2832 switch(bytes) {
2833 case 4: // fall-through
2834 case 8: // fall-through
2835 case 16: return Assembler::AVX_128bit;
2836 case 32: return Assembler::AVX_256bit;
2837 case 64: return Assembler::AVX_512bit;
2838
2839 default: {
2840 ShouldNotReachHere();
2841 return Assembler::AVX_NoVec;
2842 }
2843 }
2844 }
2845
2846 static inline Assembler::AvxVectorLen vector_length_encoding(const Node* n) {
2847 return vector_length_encoding(Matcher::vector_length_in_bytes(n));
2848 }
2849
2850 static inline Assembler::AvxVectorLen vector_length_encoding(const MachNode* use, MachOper* opnd) {
2851 uint def_idx = use->operand_index(opnd);
2852 Node* def = use->in(def_idx);
2853 return vector_length_encoding(def);
2854 }
2855
2856 static inline bool is_vector_popcount_predicate(BasicType bt) {
2857 return (is_subword_type(bt) && VM_Version::supports_avx512_bitalg()) ||
2858 (is_non_subword_integral_type(bt) && VM_Version::supports_avx512_vpopcntdq());
2859 }
2860
2861 static inline bool is_clz_non_subword_predicate_evex(BasicType bt, int vlen_bytes) {
2862 return is_non_subword_integral_type(bt) && VM_Version::supports_avx512cd() &&
2863 (VM_Version::supports_avx512vl() || vlen_bytes == 64);
2864 }
2865
2866 class Node::PD {
2867 public:
2868 enum NodeFlags : uint64_t {
2869 Flag_intel_jcc_erratum = Node::_last_flag << 1,
2870 Flag_sets_carry_flag = Node::_last_flag << 2,
2871 Flag_sets_parity_flag = Node::_last_flag << 3,
2872 Flag_sets_zero_flag = Node::_last_flag << 4,
2873 Flag_sets_overflow_flag = Node::_last_flag << 5,
2874 Flag_sets_sign_flag = Node::_last_flag << 6,
2875 Flag_clears_carry_flag = Node::_last_flag << 7,
2876 Flag_clears_parity_flag = Node::_last_flag << 8,
2877 Flag_clears_zero_flag = Node::_last_flag << 9,
2878 Flag_clears_overflow_flag = Node::_last_flag << 10,
2879 Flag_clears_sign_flag = Node::_last_flag << 11,
2880 Flag_ndd_demotable_opr1 = Node::_last_flag << 12,
2881 Flag_ndd_demotable_opr2 = Node::_last_flag << 13,
2882 _last_flag = Flag_ndd_demotable_opr2
2883 };
2884 };
2885
2886 %} // end source_hpp
2887
2888 source %{
2889
2890 #include "opto/addnode.hpp"
2891 #include "c2_intelJccErratum_x86.hpp"
2892
2893 void PhaseOutput::pd_perform_mach_node_analysis() {
2894 if (VM_Version::has_intel_jcc_erratum()) {
2895 int extra_padding = IntelJccErratum::tag_affected_machnodes(C, C->cfg(), C->regalloc());
2896 _buf_sizes._code += extra_padding;
2897 }
2898 }
2899
2900 int MachNode::pd_alignment_required() const {
2901 if (VM_Version::has_intel_jcc_erratum() && IntelJccErratum::is_jcc_erratum_branch(this)) {
2902 // Conservatively add worst case padding. We assume that relocInfo::addr_unit() is 1 on x86.
2903 return IntelJccErratum::largest_jcc_size() + 1;
2904 } else {
2905 return 1;
2906 }
2907 }
2908
2909 int MachNode::compute_padding(int current_offset) const {
2910 if (flags() & Node::PD::Flag_intel_jcc_erratum) {
2911 Compile* C = Compile::current();
2912 PhaseOutput* output = C->output();
2913 Block* block = output->block();
2914 int index = output->index();
2915 return IntelJccErratum::compute_padding(current_offset, this, block, index, C->regalloc());
2916 } else {
2917 return 0;
2918 }
2919 }
2920
2921 // Emit deopt handler code.
2922 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) {
2923
2924 // Note that the code buffer's insts_mark is always relative to insts.
2925 // That's why we must use the macroassembler to generate a handler.
2926 address base = __ start_a_stub(size_deopt_handler());
2927 if (base == nullptr) {
2928 ciEnv::current()->record_failure("CodeCache is full");
2929 return 0; // CodeBuffer::expand failed
2930 }
2931 int offset = __ offset();
2932
2933 Label start;
2934 __ bind(start);
2935
2936 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2937
2938 int entry_offset = __ offset();
2939
2940 __ jmp(start);
2941
2942 assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow %d", (__ offset() - offset));
2943 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2944 "out of bounds read in post-call NOP check");
2945 __ end_a_stub();
2946 return entry_offset;
2947 }
2948
2949 static Assembler::Width widthForType(BasicType bt) {
2950 if (bt == T_BYTE) {
2951 return Assembler::B;
2952 } else if (bt == T_SHORT) {
2953 return Assembler::W;
2954 } else if (bt == T_INT) {
2955 return Assembler::D;
2956 } else {
2957 assert(bt == T_LONG, "not a long: %s", type2name(bt));
2958 return Assembler::Q;
2959 }
2960 }
2961
2962 //=============================================================================
2963
2964 // Float masks come from different places depending on platform.
2965 static address float_signmask() { return StubRoutines::x86::float_sign_mask(); }
2966 static address float_signflip() { return StubRoutines::x86::float_sign_flip(); }
2967 static address double_signmask() { return StubRoutines::x86::double_sign_mask(); }
2968 static address double_signflip() { return StubRoutines::x86::double_sign_flip(); }
2969 static address vector_short_to_byte_mask() { return StubRoutines::x86::vector_short_to_byte_mask(); }
2970 static address vector_int_to_byte_mask() { return StubRoutines::x86::vector_int_to_byte_mask(); }
2971 static address vector_byte_perm_mask() { return StubRoutines::x86::vector_byte_perm_mask(); }
2972 static address vector_long_sign_mask() { return StubRoutines::x86::vector_long_sign_mask(); }
2973 static address vector_all_bits_set() { return StubRoutines::x86::vector_all_bits_set(); }
2974 static address vector_int_mask_cmp_bits() { return StubRoutines::x86::vector_int_mask_cmp_bits(); }
2975 static address vector_int_to_short_mask() { return StubRoutines::x86::vector_int_to_short_mask(); }
2976 static address vector_byte_shufflemask() { return StubRoutines::x86::vector_byte_shuffle_mask(); }
2977 static address vector_short_shufflemask() { return StubRoutines::x86::vector_short_shuffle_mask(); }
2978 static address vector_int_shufflemask() { return StubRoutines::x86::vector_int_shuffle_mask(); }
2979 static address vector_long_shufflemask() { return StubRoutines::x86::vector_long_shuffle_mask(); }
2980 static address vector_32_bit_mask() { return StubRoutines::x86::vector_32_bit_mask(); }
2981 static address vector_64_bit_mask() { return StubRoutines::x86::vector_64_bit_mask(); }
2982 static address vector_float_signflip() { return StubRoutines::x86::vector_float_sign_flip();}
2983 static address vector_double_signflip() { return StubRoutines::x86::vector_double_sign_flip();}
2984
2985 //=============================================================================
2986 bool Matcher::match_rule_supported(int opcode) {
2987 if (!has_match_rule(opcode)) {
2988 return false; // no match rule present
2989 }
2990 switch (opcode) {
2991 case Op_AbsVL:
2992 case Op_StoreVectorScatter:
2993 if (UseAVX < 3) {
2994 return false;
2995 }
2996 break;
2997 case Op_PopCountI:
2998 case Op_PopCountL:
2999 if (!UsePopCountInstruction) {
3000 return false;
3001 }
3002 break;
3003 case Op_PopCountVI:
3004 if (UseAVX < 2) {
3005 return false;
3006 }
3007 break;
3008 case Op_CompressV:
3009 case Op_ExpandV:
3010 case Op_PopCountVL:
3011 if (UseAVX < 2) {
3012 return false;
3013 }
3014 break;
3015 case Op_MulVI:
3016 if ((UseSSE < 4) && (UseAVX < 1)) { // only with SSE4_1 or AVX
3017 return false;
3018 }
3019 break;
3020 case Op_MulVL:
3021 if (UseSSE < 4) { // only with SSE4_1 or AVX
3022 return false;
3023 }
3024 break;
3025 case Op_MulReductionVL:
3026 if (VM_Version::supports_avx512dq() == false) {
3027 return false;
3028 }
3029 break;
3030 case Op_AbsVB:
3031 case Op_AbsVS:
3032 case Op_AbsVI:
3033 case Op_AddReductionVI:
3034 case Op_AndReductionV:
3035 case Op_OrReductionV:
3036 case Op_XorReductionV:
3037 if (UseSSE < 3) { // requires at least SSSE3
3038 return false;
3039 }
3040 break;
3041 case Op_MaxHF:
3042 case Op_MinHF:
3043 if (!VM_Version::supports_avx512vlbw()) {
3044 return false;
3045 } // fallthrough
3046 case Op_AddHF:
3047 case Op_DivHF:
3048 case Op_FmaHF:
3049 case Op_MulHF:
3050 case Op_ReinterpretS2HF:
3051 case Op_ReinterpretHF2S:
3052 case Op_SubHF:
3053 case Op_SqrtHF:
3054 if (!VM_Version::supports_avx512_fp16()) {
3055 return false;
3056 }
3057 break;
3058 case Op_VectorLoadShuffle:
3059 case Op_VectorRearrange:
3060 case Op_MulReductionVI:
3061 if (UseSSE < 4) { // requires at least SSE4
3062 return false;
3063 }
3064 break;
3065 case Op_IsInfiniteF:
3066 case Op_IsInfiniteD:
3067 if (!VM_Version::supports_avx512dq()) {
3068 return false;
3069 }
3070 break;
3071 case Op_SqrtVD:
3072 case Op_SqrtVF:
3073 case Op_VectorMaskCmp:
3074 case Op_VectorCastB2X:
3075 case Op_VectorCastS2X:
3076 case Op_VectorCastI2X:
3077 case Op_VectorCastL2X:
3078 case Op_VectorCastF2X:
3079 case Op_VectorCastD2X:
3080 case Op_VectorUCastB2X:
3081 case Op_VectorUCastS2X:
3082 case Op_VectorUCastI2X:
3083 case Op_VectorMaskCast:
3084 if (UseAVX < 1) { // enabled for AVX only
3085 return false;
3086 }
3087 break;
3088 case Op_PopulateIndex:
3089 if (UseAVX < 2) {
3090 return false;
3091 }
3092 break;
3093 case Op_RoundVF:
3094 if (UseAVX < 2) { // enabled for AVX2 only
3095 return false;
3096 }
3097 break;
3098 case Op_RoundVD:
3099 if (UseAVX < 3) {
3100 return false; // enabled for AVX3 only
3101 }
3102 break;
3103 case Op_CompareAndSwapL:
3104 case Op_CompareAndSwapP:
3105 break;
3106 case Op_StrIndexOf:
3107 if (!UseSSE42Intrinsics) {
3108 return false;
3109 }
3110 break;
3111 case Op_StrIndexOfChar:
3112 if (!UseSSE42Intrinsics) {
3113 return false;
3114 }
3115 break;
3116 case Op_OnSpinWait:
3117 if (VM_Version::supports_on_spin_wait() == false) {
3118 return false;
3119 }
3120 break;
3121 case Op_MulVB:
3122 case Op_LShiftVB:
3123 case Op_RShiftVB:
3124 case Op_URShiftVB:
3125 case Op_VectorInsert:
3126 case Op_VectorLoadMask:
3127 case Op_VectorStoreMask:
3128 case Op_VectorBlend:
3129 if (UseSSE < 4) {
3130 return false;
3131 }
3132 break;
3133 case Op_MaxD:
3134 case Op_MaxF:
3135 case Op_MinD:
3136 case Op_MinF:
3137 if (UseAVX < 1) { // enabled for AVX only
3138 return false;
3139 }
3140 break;
3141 case Op_CacheWB:
3142 case Op_CacheWBPreSync:
3143 case Op_CacheWBPostSync:
3144 if (!VM_Version::supports_data_cache_line_flush()) {
3145 return false;
3146 }
3147 break;
3148 case Op_ExtractB:
3149 case Op_ExtractL:
3150 case Op_ExtractI:
3151 case Op_RoundDoubleMode:
3152 if (UseSSE < 4) {
3153 return false;
3154 }
3155 break;
3156 case Op_RoundDoubleModeV:
3157 if (VM_Version::supports_avx() == false) {
3158 return false; // 128bit vroundpd is not available
3159 }
3160 break;
3161 case Op_LoadVectorGather:
3162 case Op_LoadVectorGatherMasked:
3163 if (UseAVX < 2) {
3164 return false;
3165 }
3166 break;
3167 case Op_FmaF:
3168 case Op_FmaD:
3169 case Op_FmaVD:
3170 case Op_FmaVF:
3171 if (!UseFMA) {
3172 return false;
3173 }
3174 break;
3175 case Op_MacroLogicV:
3176 if (UseAVX < 3 || !UseVectorMacroLogic) {
3177 return false;
3178 }
3179 break;
3180
3181 case Op_VectorCmpMasked:
3182 if (UseAVX < 3 || !UseCountTrailingZerosInstruction) {
3183 return false;
3184 }
3185 break;
3186 case Op_VectorMaskGen:
3187 if (UseAVX < 3 || !VM_Version::supports_bmi2()) {
3188 return false;
3189 }
3190 break;
3191 case Op_VectorMaskFirstTrue:
3192 case Op_VectorMaskLastTrue:
3193 case Op_VectorMaskTrueCount:
3194 case Op_VectorMaskToLong:
3195 if (UseAVX < 1) {
3196 return false;
3197 }
3198 break;
3199 case Op_RoundF:
3200 case Op_RoundD:
3201 break;
3202 case Op_CopySignD:
3203 case Op_CopySignF:
3204 if (UseAVX < 3) {
3205 return false;
3206 }
3207 if (!VM_Version::supports_avx512vl()) {
3208 return false;
3209 }
3210 break;
3211 case Op_CompressBits:
3212 case Op_ExpandBits:
3213 if (!VM_Version::supports_bmi2()) {
3214 return false;
3215 }
3216 break;
3217 case Op_CompressM:
3218 if (!VM_Version::supports_avx512vl() || !VM_Version::supports_bmi2()) {
3219 return false;
3220 }
3221 break;
3222 case Op_ConvF2HF:
3223 case Op_ConvHF2F:
3224 if (!VM_Version::supports_float16()) {
3225 return false;
3226 }
3227 break;
3228 case Op_VectorCastF2HF:
3229 case Op_VectorCastHF2F:
3230 if (!VM_Version::supports_f16c() && !VM_Version::supports_evex()) {
3231 return false;
3232 }
3233 break;
3234 }
3235 return true; // Match rules are supported by default.
3236 }
3237
3238 //------------------------------------------------------------------------
3239
3240 static inline bool is_pop_count_instr_target(BasicType bt) {
3241 return (is_subword_type(bt) && VM_Version::supports_avx512_bitalg()) ||
3242 (is_non_subword_integral_type(bt) && VM_Version::supports_avx512_vpopcntdq());
3243 }
3244
3245 bool Matcher::match_rule_supported_auto_vectorization(int opcode, int vlen, BasicType bt) {
3246 return match_rule_supported_vector(opcode, vlen, bt);
3247 }
3248
3249 // Identify extra cases that we might want to provide match rules for vector nodes and
3250 // other intrinsics guarded with vector length (vlen) and element type (bt).
3251 bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
3252 if (!match_rule_supported(opcode)) {
3253 return false;
3254 }
3255 // Matcher::vector_size_supported() restricts vector sizes in the following way (see Matcher::vector_width_in_bytes):
3256 // * SSE2 supports 128bit vectors for all types;
3257 // * AVX1 supports 256bit vectors only for FLOAT and DOUBLE types;
3258 // * AVX2 supports 256bit vectors for all types;
3259 // * AVX512F supports 512bit vectors only for INT, FLOAT, and DOUBLE types;
3260 // * AVX512BW supports 512bit vectors for BYTE, SHORT, and CHAR types.
3261 // There's also a limit on minimum vector size supported: 2 elements (or 4 bytes for BYTE).
3262 // And MaxVectorSize is taken into account as well.
3263 if (!vector_size_supported(bt, vlen)) {
3264 return false;
3265 }
3266 // Special cases which require vector length follow:
3267 // * implementation limitations
3268 // * some 512bit vector operations on FLOAT and DOUBLE types require AVX512DQ
3269 // * 128bit vroundpd instruction is present only in AVX1
3270 int size_in_bits = vlen * type2aelembytes(bt) * BitsPerByte;
3271 switch (opcode) {
3272 case Op_MaxVHF:
3273 case Op_MinVHF:
3274 if (!VM_Version::supports_avx512bw()) {
3275 return false;
3276 }
3277 case Op_AddVHF:
3278 case Op_DivVHF:
3279 case Op_FmaVHF:
3280 case Op_MulVHF:
3281 case Op_SubVHF:
3282 case Op_SqrtVHF:
3283 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3284 return false;
3285 }
3286 if (!VM_Version::supports_avx512_fp16()) {
3287 return false;
3288 }
3289 break;
3290 case Op_AbsVF:
3291 case Op_NegVF:
3292 if ((vlen == 16) && (VM_Version::supports_avx512dq() == false)) {
3293 return false; // 512bit vandps and vxorps are not available
3294 }
3295 break;
3296 case Op_AbsVD:
3297 case Op_NegVD:
3298 if ((vlen == 8) && (VM_Version::supports_avx512dq() == false)) {
3299 return false; // 512bit vpmullq, vandpd and vxorpd are not available
3300 }
3301 break;
3302 case Op_RotateRightV:
3303 case Op_RotateLeftV:
3304 if (bt != T_INT && bt != T_LONG) {
3305 return false;
3306 } // fallthrough
3307 case Op_MacroLogicV:
3308 if (!VM_Version::supports_evex() ||
3309 ((size_in_bits != 512) && !VM_Version::supports_avx512vl())) {
3310 return false;
3311 }
3312 break;
3313 case Op_ClearArray:
3314 case Op_VectorMaskGen:
3315 case Op_VectorCmpMasked:
3316 if (!VM_Version::supports_avx512bw()) {
3317 return false;
3318 }
3319 if ((size_in_bits != 512) && !VM_Version::supports_avx512vl()) {
3320 return false;
3321 }
3322 break;
3323 case Op_LoadVectorMasked:
3324 case Op_StoreVectorMasked:
3325 if (!VM_Version::supports_avx512bw() && (is_subword_type(bt) || UseAVX < 1)) {
3326 return false;
3327 }
3328 break;
3329 case Op_UMinV:
3330 case Op_UMaxV:
3331 if (UseAVX == 0) {
3332 return false;
3333 }
3334 break;
3335 case Op_UMinReductionV:
3336 case Op_UMaxReductionV:
3337 if (UseAVX == 0) {
3338 return false;
3339 }
3340 if (bt == T_LONG && !VM_Version::supports_avx512vl()) {
3341 return false;
3342 }
3343 if (UseAVX > 2 && size_in_bits == 512 && !VM_Version::supports_avx512vl()) {
3344 return false;
3345 }
3346 break;
3347 case Op_MaxV:
3348 case Op_MinV:
3349 if (UseSSE < 4 && is_integral_type(bt)) {
3350 return false;
3351 }
3352 if ((bt == T_FLOAT || bt == T_DOUBLE)) {
3353 // Float/Double intrinsics are enabled for AVX family currently.
3354 if (UseAVX == 0) {
3355 return false;
3356 }
3357 if (UseAVX > 2 && (!VM_Version::supports_avx512dq() && size_in_bits == 512)) { // 512 bit Float/Double intrinsics need AVX512DQ
3358 return false;
3359 }
3360 }
3361 break;
3362 case Op_CallLeafVector:
3363 if (size_in_bits == 512 && !VM_Version::supports_avx512vlbwdq()) {
3364 return false;
3365 }
3366 break;
3367 case Op_AddReductionVI:
3368 if (bt == T_INT && (UseSSE < 3 || !VM_Version::supports_ssse3())) {
3369 return false;
3370 }
3371 // fallthrough
3372 case Op_AndReductionV:
3373 case Op_OrReductionV:
3374 case Op_XorReductionV:
3375 if (is_subword_type(bt) && (UseSSE < 4)) {
3376 return false;
3377 }
3378 break;
3379 case Op_MinReductionV:
3380 case Op_MaxReductionV:
3381 if ((bt == T_INT || is_subword_type(bt)) && UseSSE < 4) {
3382 return false;
3383 } else if (bt == T_LONG && (UseAVX < 3 || !VM_Version::supports_avx512vlbwdq())) {
3384 return false;
3385 }
3386 // Float/Double intrinsics enabled for AVX family.
3387 if (UseAVX == 0 && (bt == T_FLOAT || bt == T_DOUBLE)) {
3388 return false;
3389 }
3390 if (UseAVX > 2 && (!VM_Version::supports_avx512dq() && size_in_bits == 512)) {
3391 return false;
3392 }
3393 break;
3394 case Op_VectorBlend:
3395 if (UseAVX == 0 && size_in_bits < 128) {
3396 return false;
3397 }
3398 break;
3399 case Op_VectorTest:
3400 if (UseSSE < 4) {
3401 return false; // Implementation limitation
3402 } else if (size_in_bits < 32) {
3403 return false; // Implementation limitation
3404 }
3405 break;
3406 case Op_VectorLoadShuffle:
3407 case Op_VectorRearrange:
3408 if(vlen == 2) {
3409 return false; // Implementation limitation due to how shuffle is loaded
3410 } else if (size_in_bits == 256 && UseAVX < 2) {
3411 return false; // Implementation limitation
3412 }
3413 break;
3414 case Op_VectorLoadMask:
3415 case Op_VectorMaskCast:
3416 if (size_in_bits == 256 && UseAVX < 2) {
3417 return false; // Implementation limitation
3418 }
3419 // fallthrough
3420 case Op_VectorStoreMask:
3421 if (vlen == 2) {
3422 return false; // Implementation limitation
3423 }
3424 break;
3425 case Op_PopulateIndex:
3426 if (size_in_bits > 256 && !VM_Version::supports_avx512bw()) {
3427 return false;
3428 }
3429 break;
3430 case Op_VectorCastB2X:
3431 case Op_VectorCastS2X:
3432 case Op_VectorCastI2X:
3433 if (bt != T_DOUBLE && size_in_bits == 256 && UseAVX < 2) {
3434 return false;
3435 }
3436 break;
3437 case Op_VectorCastL2X:
3438 if (is_integral_type(bt) && size_in_bits == 256 && UseAVX < 2) {
3439 return false;
3440 } else if (!is_integral_type(bt) && !VM_Version::supports_avx512dq()) {
3441 return false;
3442 }
3443 break;
3444 case Op_VectorCastF2X: {
3445 // As per JLS section 5.1.3 narrowing conversion to sub-word types
3446 // happen after intermediate conversion to integer and special handling
3447 // code needs AVX2 vpcmpeqd instruction for 256 bit vectors.
3448 int src_size_in_bits = type2aelembytes(T_FLOAT) * vlen * BitsPerByte;
3449 if (is_integral_type(bt) && src_size_in_bits == 256 && UseAVX < 2) {
3450 return false;
3451 }
3452 }
3453 // fallthrough
3454 case Op_VectorCastD2X:
3455 if (bt == T_LONG && !VM_Version::supports_avx512dq()) {
3456 return false;
3457 }
3458 break;
3459 case Op_VectorCastF2HF:
3460 case Op_VectorCastHF2F:
3461 if (!VM_Version::supports_f16c() &&
3462 ((!VM_Version::supports_evex() ||
3463 ((size_in_bits != 512) && !VM_Version::supports_avx512vl())))) {
3464 return false;
3465 }
3466 break;
3467 case Op_RoundVD:
3468 if (!VM_Version::supports_avx512dq()) {
3469 return false;
3470 }
3471 break;
3472 case Op_MulReductionVI:
3473 if (bt == T_BYTE && size_in_bits == 512 && !VM_Version::supports_avx512bw()) {
3474 return false;
3475 }
3476 break;
3477 case Op_LoadVectorGatherMasked:
3478 if (!is_subword_type(bt) && size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3479 return false;
3480 }
3481 if (is_subword_type(bt) &&
3482 ((size_in_bits > 256 && !VM_Version::supports_avx512bw()) ||
3483 (size_in_bits < 64) ||
3484 (bt == T_SHORT && !VM_Version::supports_bmi2()))) {
3485 return false;
3486 }
3487 break;
3488 case Op_StoreVectorScatterMasked:
3489 case Op_StoreVectorScatter:
3490 if (is_subword_type(bt)) {
3491 return false;
3492 } else if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3493 return false;
3494 }
3495 // fallthrough
3496 case Op_LoadVectorGather:
3497 if (!is_subword_type(bt) && size_in_bits == 64) {
3498 return false;
3499 }
3500 if (is_subword_type(bt) && size_in_bits < 64) {
3501 return false;
3502 }
3503 break;
3504 case Op_SaturatingAddV:
3505 case Op_SaturatingSubV:
3506 if (UseAVX < 1) {
3507 return false; // Implementation limitation
3508 }
3509 if (is_subword_type(bt) && size_in_bits == 512 && !VM_Version::supports_avx512bw()) {
3510 return false;
3511 }
3512 break;
3513 case Op_SelectFromTwoVector:
3514 if (size_in_bits < 128) {
3515 return false;
3516 }
3517 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3518 return false;
3519 }
3520 if (bt == T_SHORT && !VM_Version::supports_avx512bw()) {
3521 return false;
3522 }
3523 if (bt == T_BYTE && !VM_Version::supports_avx512_vbmi()) {
3524 return false;
3525 }
3526 if ((bt == T_INT || bt == T_FLOAT || bt == T_DOUBLE) && !VM_Version::supports_evex()) {
3527 return false;
3528 }
3529 break;
3530 case Op_MaskAll:
3531 if (!VM_Version::supports_evex()) {
3532 return false;
3533 }
3534 if ((vlen > 16 || is_subword_type(bt)) && !VM_Version::supports_avx512bw()) {
3535 return false;
3536 }
3537 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3538 return false;
3539 }
3540 break;
3541 case Op_VectorMaskCmp:
3542 if (vlen < 2 || size_in_bits < 32) {
3543 return false;
3544 }
3545 break;
3546 case Op_CompressM:
3547 if (UseAVX < 3 || !VM_Version::supports_bmi2()) {
3548 return false;
3549 }
3550 break;
3551 case Op_CompressV:
3552 case Op_ExpandV:
3553 if (is_subword_type(bt) && !VM_Version::supports_avx512_vbmi2()) {
3554 return false;
3555 }
3556 if (size_in_bits < 128 ) {
3557 return false;
3558 }
3559 case Op_VectorLongToMask:
3560 if (UseAVX < 1) {
3561 return false;
3562 }
3563 if (UseAVX < 3 && !VM_Version::supports_bmi2()) {
3564 return false;
3565 }
3566 break;
3567 case Op_SignumVD:
3568 case Op_SignumVF:
3569 if (UseAVX < 1) {
3570 return false;
3571 }
3572 break;
3573 case Op_PopCountVI:
3574 case Op_PopCountVL: {
3575 if (!is_pop_count_instr_target(bt) &&
3576 (size_in_bits == 512) && !VM_Version::supports_avx512bw()) {
3577 return false;
3578 }
3579 }
3580 break;
3581 case Op_ReverseV:
3582 case Op_ReverseBytesV:
3583 if (UseAVX < 2) {
3584 return false;
3585 }
3586 break;
3587 case Op_CountTrailingZerosV:
3588 case Op_CountLeadingZerosV:
3589 if (UseAVX < 2) {
3590 return false;
3591 }
3592 break;
3593 }
3594 return true; // Per default match rules are supported.
3595 }
3596
3597 bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
3598 // ADLC based match_rule_supported routine checks for the existence of pattern based
3599 // on IR opcode. Most of the unary/binary/ternary masked operation share the IR nodes
3600 // of their non-masked counterpart with mask edge being the differentiator.
3601 // This routine does a strict check on the existence of masked operation patterns
3602 // by returning a default false value for all the other opcodes apart from the
3603 // ones whose masked instruction patterns are defined in this file.
3604 if (!match_rule_supported_vector(opcode, vlen, bt)) {
3605 return false;
3606 }
3607
3608 int size_in_bits = vlen * type2aelembytes(bt) * BitsPerByte;
3609 if (size_in_bits != 512 && !VM_Version::supports_avx512vl()) {
3610 return false;
3611 }
3612 switch(opcode) {
3613 // Unary masked operations
3614 case Op_AbsVB:
3615 case Op_AbsVS:
3616 if(!VM_Version::supports_avx512bw()) {
3617 return false; // Implementation limitation
3618 }
3619 case Op_AbsVI:
3620 case Op_AbsVL:
3621 return true;
3622
3623 // Ternary masked operations
3624 case Op_FmaVF:
3625 case Op_FmaVD:
3626 return true;
3627
3628 case Op_MacroLogicV:
3629 if(bt != T_INT && bt != T_LONG) {
3630 return false;
3631 }
3632 return true;
3633
3634 // Binary masked operations
3635 case Op_AddVB:
3636 case Op_AddVS:
3637 case Op_SubVB:
3638 case Op_SubVS:
3639 case Op_MulVS:
3640 case Op_LShiftVS:
3641 case Op_RShiftVS:
3642 case Op_URShiftVS:
3643 assert(size_in_bits == 512 || VM_Version::supports_avx512vl(), "");
3644 if (!VM_Version::supports_avx512bw()) {
3645 return false; // Implementation limitation
3646 }
3647 return true;
3648
3649 case Op_MulVL:
3650 assert(size_in_bits == 512 || VM_Version::supports_avx512vl(), "");
3651 if (!VM_Version::supports_avx512dq()) {
3652 return false; // Implementation limitation
3653 }
3654 return true;
3655
3656 case Op_AndV:
3657 case Op_OrV:
3658 case Op_XorV:
3659 case Op_RotateRightV:
3660 case Op_RotateLeftV:
3661 if (bt != T_INT && bt != T_LONG) {
3662 return false; // Implementation limitation
3663 }
3664 return true;
3665
3666 case Op_VectorLoadMask:
3667 assert(size_in_bits == 512 || VM_Version::supports_avx512vl(), "");
3668 if (is_subword_type(bt) && !VM_Version::supports_avx512bw()) {
3669 return false;
3670 }
3671 return true;
3672
3673 case Op_AddVI:
3674 case Op_AddVL:
3675 case Op_AddVF:
3676 case Op_AddVD:
3677 case Op_SubVI:
3678 case Op_SubVL:
3679 case Op_SubVF:
3680 case Op_SubVD:
3681 case Op_MulVI:
3682 case Op_MulVF:
3683 case Op_MulVD:
3684 case Op_DivVF:
3685 case Op_DivVD:
3686 case Op_SqrtVF:
3687 case Op_SqrtVD:
3688 case Op_LShiftVI:
3689 case Op_LShiftVL:
3690 case Op_RShiftVI:
3691 case Op_RShiftVL:
3692 case Op_URShiftVI:
3693 case Op_URShiftVL:
3694 case Op_LoadVectorMasked:
3695 case Op_StoreVectorMasked:
3696 case Op_LoadVectorGatherMasked:
3697 case Op_StoreVectorScatterMasked:
3698 return true;
3699
3700 case Op_UMinV:
3701 case Op_UMaxV:
3702 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3703 return false;
3704 } // fallthrough
3705 case Op_MaxV:
3706 case Op_MinV:
3707 if (is_subword_type(bt) && !VM_Version::supports_avx512bw()) {
3708 return false; // Implementation limitation
3709 }
3710 if (is_floating_point_type(bt) && !VM_Version::supports_avx10_2()) {
3711 return false; // Implementation limitation
3712 }
3713 return true;
3714 case Op_SaturatingAddV:
3715 case Op_SaturatingSubV:
3716 if (!is_subword_type(bt)) {
3717 return false;
3718 }
3719 if (size_in_bits < 128 || !VM_Version::supports_avx512bw()) {
3720 return false; // Implementation limitation
3721 }
3722 return true;
3723
3724 case Op_VectorMaskCmp:
3725 if (is_subword_type(bt) && !VM_Version::supports_avx512bw()) {
3726 return false; // Implementation limitation
3727 }
3728 return true;
3729
3730 case Op_VectorRearrange:
3731 if (bt == T_SHORT && !VM_Version::supports_avx512bw()) {
3732 return false; // Implementation limitation
3733 }
3734 if (bt == T_BYTE && !VM_Version::supports_avx512_vbmi()) {
3735 return false; // Implementation limitation
3736 } else if ((bt == T_INT || bt == T_FLOAT) && size_in_bits < 256) {
3737 return false; // Implementation limitation
3738 }
3739 return true;
3740
3741 // Binary Logical operations
3742 case Op_AndVMask:
3743 case Op_OrVMask:
3744 case Op_XorVMask:
3745 if (vlen > 16 && !VM_Version::supports_avx512bw()) {
3746 return false; // Implementation limitation
3747 }
3748 return true;
3749
3750 case Op_PopCountVI:
3751 case Op_PopCountVL:
3752 if (!is_pop_count_instr_target(bt)) {
3753 return false;
3754 }
3755 return true;
3756
3757 case Op_MaskAll:
3758 return true;
3759
3760 case Op_CountLeadingZerosV:
3761 if (is_non_subword_integral_type(bt) && VM_Version::supports_avx512cd()) {
3762 return true;
3763 }
3764 default:
3765 return false;
3766 }
3767 }
3768
3769 bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* vt) {
3770 return false;
3771 }
3772
3773 // Return true if Vector::rearrange needs preparation of the shuffle argument
3774 bool Matcher::vector_rearrange_requires_load_shuffle(BasicType elem_bt, int vlen) {
3775 switch (elem_bt) {
3776 case T_BYTE: return false;
3777 case T_SHORT: return !VM_Version::supports_avx512bw();
3778 case T_INT: return !VM_Version::supports_avx();
3779 case T_LONG: return vlen < 8 && !VM_Version::supports_avx512vl();
3780 default:
3781 ShouldNotReachHere();
3782 return false;
3783 }
3784 }
3785
3786 bool Matcher::mask_op_prefers_predicate(int opcode, const TypeVect* vt) {
3787 // Prefer predicate if the mask type is "TypePVectMask".
3788 return vt->isa_pvectmask() != nullptr;
3789 }
3790
3791 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
3792 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
3793 bool legacy = (generic_opnd->opcode() == LEGVEC);
3794 if (!VM_Version::supports_avx512vlbwdq() && // KNL
3795 is_temp && !legacy && (ideal_reg == Op_VecZ)) {
3796 // Conservatively specialize 512bit vec TEMP operands to legVecZ (zmm0-15) on KNL.
3797 return new legVecZOper();
3798 }
3799 if (legacy) {
3800 switch (ideal_reg) {
3801 case Op_VecS: return new legVecSOper();
3802 case Op_VecD: return new legVecDOper();
3803 case Op_VecX: return new legVecXOper();
3804 case Op_VecY: return new legVecYOper();
3805 case Op_VecZ: return new legVecZOper();
3806 }
3807 } else {
3808 switch (ideal_reg) {
3809 case Op_VecS: return new vecSOper();
3810 case Op_VecD: return new vecDOper();
3811 case Op_VecX: return new vecXOper();
3812 case Op_VecY: return new vecYOper();
3813 case Op_VecZ: return new vecZOper();
3814 }
3815 }
3816 ShouldNotReachHere();
3817 return nullptr;
3818 }
3819
3820 bool Matcher::is_reg2reg_move(MachNode* m) {
3821 switch (m->rule()) {
3822 case MoveVec2Leg_rule:
3823 case MoveLeg2Vec_rule:
3824 case MoveF2VL_rule:
3825 case MoveF2LEG_rule:
3826 case MoveVL2F_rule:
3827 case MoveLEG2F_rule:
3828 case MoveD2VL_rule:
3829 case MoveD2LEG_rule:
3830 case MoveVL2D_rule:
3831 case MoveLEG2D_rule:
3832 return true;
3833 default:
3834 return false;
3835 }
3836 }
3837
3838 bool Matcher::is_generic_vector(MachOper* opnd) {
3839 switch (opnd->opcode()) {
3840 case VEC:
3841 case LEGVEC:
3842 return true;
3843 default:
3844 return false;
3845 }
3846 }
3847
3848 //------------------------------------------------------------------------
3849
3850 const RegMask* Matcher::predicate_reg_mask(void) {
3851 return &_VECTMASK_REG_mask;
3852 }
3853
3854 // Max vector size in bytes. 0 if not supported.
3855 int Matcher::vector_width_in_bytes(BasicType bt) {
3856 assert(is_java_primitive(bt), "only primitive type vectors");
3857 // SSE2 supports 128bit vectors for all types.
3858 // AVX2 supports 256bit vectors for all types.
3859 // AVX2/EVEX supports 512bit vectors for all types.
3860 int size = (UseAVX > 1) ? (1 << UseAVX) * 8 : 16;
3861 // AVX1 supports 256bit vectors only for FLOAT and DOUBLE.
3862 if (UseAVX > 0 && (bt == T_FLOAT || bt == T_DOUBLE))
3863 size = (UseAVX > 2) ? 64 : 32;
3864 if (UseAVX > 2 && (bt == T_BYTE || bt == T_SHORT || bt == T_CHAR))
3865 size = (VM_Version::supports_avx512bw()) ? 64 : 32;
3866 // Use flag to limit vector size.
3867 size = MIN2(size,(int)MaxVectorSize);
3868 // Minimum 2 values in vector (or 4 for bytes).
3869 switch (bt) {
3870 case T_DOUBLE:
3871 case T_LONG:
3872 if (size < 16) return 0;
3873 break;
3874 case T_FLOAT:
3875 case T_INT:
3876 if (size < 8) return 0;
3877 break;
3878 case T_BOOLEAN:
3879 if (size < 4) return 0;
3880 break;
3881 case T_CHAR:
3882 if (size < 4) return 0;
3883 break;
3884 case T_BYTE:
3885 if (size < 4) return 0;
3886 break;
3887 case T_SHORT:
3888 if (size < 4) return 0;
3889 break;
3890 default:
3891 ShouldNotReachHere();
3892 }
3893 return size;
3894 }
3895
3896 // Limits on vector size (number of elements) loaded into vector.
3897 int Matcher::max_vector_size(const BasicType bt) {
3898 return vector_width_in_bytes(bt)/type2aelembytes(bt);
3899 }
3900 int Matcher::min_vector_size(const BasicType bt) {
3901 int max_size = max_vector_size(bt);
3902 // Min size which can be loaded into vector is 4 bytes.
3903 int size = (type2aelembytes(bt) == 1) ? 4 : 2;
3904 // Support for calling svml double64 vectors
3905 if (bt == T_DOUBLE) {
3906 size = 1;
3907 }
3908 return MIN2(size,max_size);
3909 }
3910
3911 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
3912 // Limit the max vector size for auto vectorization to 256 bits (32 bytes)
3913 // by default on Cascade Lake
3914 if (VM_Version::is_default_intel_cascade_lake()) {
3915 return MIN2(Matcher::max_vector_size(bt), 32 / type2aelembytes(bt));
3916 }
3917 return Matcher::max_vector_size(bt);
3918 }
3919
3920 int Matcher::scalable_vector_reg_size(const BasicType bt) {
3921 return -1;
3922 }
3923
3924 // Vector ideal reg corresponding to specified size in bytes
3925 uint Matcher::vector_ideal_reg(int size) {
3926 assert(MaxVectorSize >= size, "");
3927 switch(size) {
3928 case 4: return Op_VecS;
3929 case 8: return Op_VecD;
3930 case 16: return Op_VecX;
3931 case 32: return Op_VecY;
3932 case 64: return Op_VecZ;
3933 }
3934 ShouldNotReachHere();
3935 return 0;
3936 }
3937
3938 // Check for shift by small constant as well
3939 static bool clone_shift(Node* shift, Matcher* matcher, Matcher::MStack& mstack, VectorSet& address_visited) {
3940 if (shift->Opcode() == Op_LShiftX && shift->in(2)->is_Con() &&
3941 shift->in(2)->get_int() <= 3 &&
3942 // Are there other uses besides address expressions?
3943 !matcher->is_visited(shift)) {
3944 address_visited.set(shift->_idx); // Flag as address_visited
3945 mstack.push(shift->in(2), Matcher::Visit);
3946 Node *conv = shift->in(1);
3947 // Allow Matcher to match the rule which bypass
3948 // ConvI2L operation for an array index on LP64
3949 // if the index value is positive.
3950 if (conv->Opcode() == Op_ConvI2L &&
3951 conv->as_Type()->type()->is_long()->_lo >= 0 &&
3952 // Are there other uses besides address expressions?
3953 !matcher->is_visited(conv)) {
3954 address_visited.set(conv->_idx); // Flag as address_visited
3955 mstack.push(conv->in(1), Matcher::Pre_Visit);
3956 } else {
3957 mstack.push(conv, Matcher::Pre_Visit);
3958 }
3959 return true;
3960 }
3961 return false;
3962 }
3963
3964 // This function identifies sub-graphs in which a 'load' node is
3965 // input to two different nodes, and such that it can be matched
3966 // with BMI instructions like blsi, blsr, etc.
3967 // Example : for b = -a[i] & a[i] can be matched to blsi r32, m32.
3968 // The graph is (AndL (SubL Con0 LoadL*) LoadL*), where LoadL*
3969 // refers to the same node.
3970 //
3971 // Match the generic fused operations pattern (op1 (op2 Con{ConType} mop) mop)
3972 // This is a temporary solution until we make DAGs expressible in ADL.
3973 template<typename ConType>
3974 class FusedPatternMatcher {
3975 Node* _op1_node;
3976 Node* _mop_node;
3977 int _con_op;
3978
3979 static int match_next(Node* n, int next_op, int next_op_idx) {
3980 if (n->in(1) == nullptr || n->in(2) == nullptr) {
3981 return -1;
3982 }
3983
3984 if (next_op_idx == -1) { // n is commutative, try rotations
3985 if (n->in(1)->Opcode() == next_op) {
3986 return 1;
3987 } else if (n->in(2)->Opcode() == next_op) {
3988 return 2;
3989 }
3990 } else {
3991 assert(next_op_idx > 0 && next_op_idx <= 2, "Bad argument index");
3992 if (n->in(next_op_idx)->Opcode() == next_op) {
3993 return next_op_idx;
3994 }
3995 }
3996 return -1;
3997 }
3998
3999 public:
4000 FusedPatternMatcher(Node* op1_node, Node* mop_node, int con_op) :
4001 _op1_node(op1_node), _mop_node(mop_node), _con_op(con_op) { }
4002
4003 bool match(int op1, int op1_op2_idx, // op1 and the index of the op1->op2 edge, -1 if op1 is commutative
4004 int op2, int op2_con_idx, // op2 and the index of the op2->con edge, -1 if op2 is commutative
4005 typename ConType::NativeType con_value) {
4006 if (_op1_node->Opcode() != op1) {
4007 return false;
4008 }
4009 if (_mop_node->outcnt() > 2) {
4010 return false;
4011 }
4012 op1_op2_idx = match_next(_op1_node, op2, op1_op2_idx);
4013 if (op1_op2_idx == -1) {
4014 return false;
4015 }
4016 // Memory operation must be the other edge
4017 int op1_mop_idx = (op1_op2_idx & 1) + 1;
4018
4019 // Check that the mop node is really what we want
4020 if (_op1_node->in(op1_mop_idx) == _mop_node) {
4021 Node* op2_node = _op1_node->in(op1_op2_idx);
4022 if (op2_node->outcnt() > 1) {
4023 return false;
4024 }
4025 assert(op2_node->Opcode() == op2, "Should be");
4026 op2_con_idx = match_next(op2_node, _con_op, op2_con_idx);
4027 if (op2_con_idx == -1) {
4028 return false;
4029 }
4030 // Memory operation must be the other edge
4031 int op2_mop_idx = (op2_con_idx & 1) + 1;
4032 // Check that the memory operation is the same node
4033 if (op2_node->in(op2_mop_idx) == _mop_node) {
4034 // Now check the constant
4035 const Type* con_type = op2_node->in(op2_con_idx)->bottom_type();
4036 if (con_type != Type::TOP && ConType::as_self(con_type)->get_con() == con_value) {
4037 return true;
4038 }
4039 }
4040 }
4041 return false;
4042 }
4043 };
4044
4045 static bool is_bmi_pattern(Node* n, Node* m) {
4046 assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "sanity");
4047 if (n != nullptr && m != nullptr) {
4048 if (m->Opcode() == Op_LoadI) {
4049 FusedPatternMatcher<TypeInt> bmii(n, m, Op_ConI);
4050 return bmii.match(Op_AndI, -1, Op_SubI, 1, 0) ||
4051 bmii.match(Op_AndI, -1, Op_AddI, -1, -1) ||
4052 bmii.match(Op_XorI, -1, Op_AddI, -1, -1);
4053 } else if (m->Opcode() == Op_LoadL) {
4054 FusedPatternMatcher<TypeLong> bmil(n, m, Op_ConL);
4055 return bmil.match(Op_AndL, -1, Op_SubL, 1, 0) ||
4056 bmil.match(Op_AndL, -1, Op_AddL, -1, -1) ||
4057 bmil.match(Op_XorL, -1, Op_AddL, -1, -1);
4058 }
4059 }
4060 return false;
4061 }
4062
4063 // Should the matcher clone input 'm' of node 'n'?
4064 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
4065 // If 'n' and 'm' are part of a graph for BMI instruction, clone the input 'm'.
4066 if (VM_Version::supports_bmi1() && VM_Version::supports_avx() && is_bmi_pattern(n, m)) {
4067 mstack.push(m, Visit);
4068 return true;
4069 }
4070 if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
4071 mstack.push(m, Visit); // m = ShiftCntV
4072 return true;
4073 }
4074 if (is_encode_and_store_pattern(n, m)) {
4075 mstack.push(m, Visit);
4076 return true;
4077 }
4078 return false;
4079 }
4080
4081 // Should the Matcher clone shifts on addressing modes, expecting them
4082 // to be subsumed into complex addressing expressions or compute them
4083 // into registers?
4084 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
4085 Node *off = m->in(AddPNode::Offset);
4086 if (off->is_Con()) {
4087 address_visited.test_set(m->_idx); // Flag as address_visited
4088 Node *adr = m->in(AddPNode::Address);
4089
4090 // Intel can handle 2 adds in addressing mode, with one of them using an immediate offset.
4091 // AtomicAdd is not an addressing expression.
4092 // Cheap to find it by looking for screwy base.
4093 if (adr->is_AddP() &&
4094 !adr->in(AddPNode::Base)->is_top() &&
4095 !adr->in(AddPNode::Offset)->is_Con() &&
4096 off->get_long() == (int) (off->get_long()) && // immL32
4097 // Are there other uses besides address expressions?
4098 !is_visited(adr)) {
4099 address_visited.set(adr->_idx); // Flag as address_visited
4100 Node *shift = adr->in(AddPNode::Offset);
4101 if (!clone_shift(shift, this, mstack, address_visited)) {
4102 mstack.push(shift, Pre_Visit);
4103 }
4104 mstack.push(adr->in(AddPNode::Address), Pre_Visit);
4105 mstack.push(adr->in(AddPNode::Base), Pre_Visit);
4106 } else {
4107 mstack.push(adr, Pre_Visit);
4108 }
4109
4110 // Clone X+offset as it also folds into most addressing expressions
4111 mstack.push(off, Visit);
4112 mstack.push(m->in(AddPNode::Base), Pre_Visit);
4113 return true;
4114 } else if (clone_shift(off, this, mstack, address_visited)) {
4115 address_visited.test_set(m->_idx); // Flag as address_visited
4116 mstack.push(m->in(AddPNode::Address), Pre_Visit);
4117 mstack.push(m->in(AddPNode::Base), Pre_Visit);
4118 return true;
4119 }
4120 return false;
4121 }
4122
4123 static inline Assembler::ComparisonPredicate booltest_pred_to_comparison_pred(int bt) {
4124 switch (bt) {
4125 case BoolTest::eq:
4126 return Assembler::eq;
4127 case BoolTest::ne:
4128 return Assembler::neq;
4129 case BoolTest::le:
4130 case BoolTest::ule:
4131 return Assembler::le;
4132 case BoolTest::ge:
4133 case BoolTest::uge:
4134 return Assembler::nlt;
4135 case BoolTest::lt:
4136 case BoolTest::ult:
4137 return Assembler::lt;
4138 case BoolTest::gt:
4139 case BoolTest::ugt:
4140 return Assembler::nle;
4141 default : ShouldNotReachHere(); return Assembler::_false;
4142 }
4143 }
4144
4145 static inline Assembler::ComparisonPredicateFP booltest_pred_to_comparison_pred_fp(int bt) {
4146 switch (bt) {
4147 case BoolTest::eq: return Assembler::EQ_OQ; // ordered non-signaling
4148 // As per JLS 15.21.1, != of NaNs is true. Thus use unordered compare.
4149 case BoolTest::ne: return Assembler::NEQ_UQ; // unordered non-signaling
4150 case BoolTest::le: return Assembler::LE_OQ; // ordered non-signaling
4151 case BoolTest::ge: return Assembler::GE_OQ; // ordered non-signaling
4152 case BoolTest::lt: return Assembler::LT_OQ; // ordered non-signaling
4153 case BoolTest::gt: return Assembler::GT_OQ; // ordered non-signaling
4154 default: ShouldNotReachHere(); return Assembler::FALSE_OS;
4155 }
4156 }
4157
4158 // Helper methods for MachSpillCopyNode::implementation().
4159 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
4160 int src_hi, int dst_hi, uint ireg, outputStream* st) {
4161 assert(ireg == Op_VecS || // 32bit vector
4162 ((src_lo & 1) == 0 && (src_lo + 1) == src_hi &&
4163 (dst_lo & 1) == 0 && (dst_lo + 1) == dst_hi),
4164 "no non-adjacent vector moves" );
4165 if (masm) {
4166 switch (ireg) {
4167 case Op_VecS: // copy whole register
4168 case Op_VecD:
4169 case Op_VecX:
4170 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4171 __ movdqu(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]));
4172 } else {
4173 __ vextractf32x4(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]), 0x0);
4174 }
4175 break;
4176 case Op_VecY:
4177 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4178 __ vmovdqu(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]));
4179 } else {
4180 __ vextractf64x4(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]), 0x0);
4181 }
4182 break;
4183 case Op_VecZ:
4184 __ evmovdquq(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]), 2);
4185 break;
4186 default:
4187 ShouldNotReachHere();
4188 }
4189 #ifndef PRODUCT
4190 } else {
4191 switch (ireg) {
4192 case Op_VecS:
4193 case Op_VecD:
4194 case Op_VecX:
4195 st->print("movdqu %s,%s\t# spill",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
4196 break;
4197 case Op_VecY:
4198 case Op_VecZ:
4199 st->print("vmovdqu %s,%s\t# spill",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
4200 break;
4201 default:
4202 ShouldNotReachHere();
4203 }
4204 #endif
4205 }
4206 }
4207
4208 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
4209 int stack_offset, int reg, uint ireg, outputStream* st) {
4210 if (masm) {
4211 if (is_load) {
4212 switch (ireg) {
4213 case Op_VecS:
4214 __ movdl(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4215 break;
4216 case Op_VecD:
4217 __ movq(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4218 break;
4219 case Op_VecX:
4220 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4221 __ movdqu(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4222 } else {
4223 __ vpxor(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), 2);
4224 __ vinsertf32x4(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset),0x0);
4225 }
4226 break;
4227 case Op_VecY:
4228 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4229 __ vmovdqu(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4230 } else {
4231 __ vpxor(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), 2);
4232 __ vinsertf64x4(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset),0x0);
4233 }
4234 break;
4235 case Op_VecZ:
4236 __ evmovdquq(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset), 2);
4237 break;
4238 default:
4239 ShouldNotReachHere();
4240 }
4241 } else { // store
4242 switch (ireg) {
4243 case Op_VecS:
4244 __ movdl(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4245 break;
4246 case Op_VecD:
4247 __ movq(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4248 break;
4249 case Op_VecX:
4250 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4251 __ movdqu(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4252 }
4253 else {
4254 __ vextractf32x4(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]), 0x0);
4255 }
4256 break;
4257 case Op_VecY:
4258 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4259 __ vmovdqu(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4260 }
4261 else {
4262 __ vextractf64x4(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]), 0x0);
4263 }
4264 break;
4265 case Op_VecZ:
4266 __ evmovdquq(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]), 2);
4267 break;
4268 default:
4269 ShouldNotReachHere();
4270 }
4271 }
4272 #ifndef PRODUCT
4273 } else {
4274 if (is_load) {
4275 switch (ireg) {
4276 case Op_VecS:
4277 st->print("movd %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4278 break;
4279 case Op_VecD:
4280 st->print("movq %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4281 break;
4282 case Op_VecX:
4283 st->print("movdqu %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4284 break;
4285 case Op_VecY:
4286 case Op_VecZ:
4287 st->print("vmovdqu %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4288 break;
4289 default:
4290 ShouldNotReachHere();
4291 }
4292 } else { // store
4293 switch (ireg) {
4294 case Op_VecS:
4295 st->print("movd [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4296 break;
4297 case Op_VecD:
4298 st->print("movq [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4299 break;
4300 case Op_VecX:
4301 st->print("movdqu [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4302 break;
4303 case Op_VecY:
4304 case Op_VecZ:
4305 st->print("vmovdqu [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4306 break;
4307 default:
4308 ShouldNotReachHere();
4309 }
4310 }
4311 #endif
4312 }
4313 }
4314
4315 template <class T>
4316 static inline GrowableArray<jbyte>* vreplicate_imm(BasicType bt, T con, int len) {
4317 int size = type2aelembytes(bt) * len;
4318 GrowableArray<jbyte>* val = new GrowableArray<jbyte>(size, size, 0);
4319 for (int i = 0; i < len; i++) {
4320 int offset = i * type2aelembytes(bt);
4321 switch (bt) {
4322 case T_BYTE: val->at(i) = con; break;
4323 case T_SHORT: {
4324 jshort c = con;
4325 memcpy(val->adr_at(offset), &c, sizeof(jshort));
4326 break;
4327 }
4328 case T_INT: {
4329 jint c = con;
4330 memcpy(val->adr_at(offset), &c, sizeof(jint));
4331 break;
4332 }
4333 case T_LONG: {
4334 jlong c = con;
4335 memcpy(val->adr_at(offset), &c, sizeof(jlong));
4336 break;
4337 }
4338 case T_FLOAT: {
4339 jfloat c = con;
4340 memcpy(val->adr_at(offset), &c, sizeof(jfloat));
4341 break;
4342 }
4343 case T_DOUBLE: {
4344 jdouble c = con;
4345 memcpy(val->adr_at(offset), &c, sizeof(jdouble));
4346 break;
4347 }
4348 default: assert(false, "%s", type2name(bt));
4349 }
4350 }
4351 return val;
4352 }
4353
4354 static inline jlong high_bit_set(BasicType bt) {
4355 switch (bt) {
4356 case T_BYTE: return 0x8080808080808080;
4357 case T_SHORT: return 0x8000800080008000;
4358 case T_INT: return 0x8000000080000000;
4359 case T_LONG: return 0x8000000000000000;
4360 default:
4361 ShouldNotReachHere();
4362 return 0;
4363 }
4364 }
4365
4366 #ifndef PRODUCT
4367 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
4368 st->print("nop \t# %d bytes pad for loops and calls", _count);
4369 }
4370 #endif
4371
4372 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
4373 __ nop(_count);
4374 }
4375
4376 uint MachNopNode::size(PhaseRegAlloc*) const {
4377 return _count;
4378 }
4379
4380 #ifndef PRODUCT
4381 void MachBreakpointNode::format(PhaseRegAlloc*, outputStream* st) const {
4382 st->print("# breakpoint");
4383 }
4384 #endif
4385
4386 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const {
4387 __ int3();
4388 }
4389
4390 uint MachBreakpointNode::size(PhaseRegAlloc* ra_) const {
4391 return MachNode::size(ra_);
4392 }
4393
4394 %}
4395
4396 //----------ENCODING BLOCK-----------------------------------------------------
4397 // This block specifies the encoding classes used by the compiler to
4398 // output byte streams. Encoding classes are parameterized macros
4399 // used by Machine Instruction Nodes in order to generate the bit
4400 // encoding of the instruction. Operands specify their base encoding
4401 // interface with the interface keyword. There are currently
4402 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
4403 // COND_INTER. REG_INTER causes an operand to generate a function
4404 // which returns its register number when queried. CONST_INTER causes
4405 // an operand to generate a function which returns the value of the
4406 // constant when queried. MEMORY_INTER causes an operand to generate
4407 // four functions which return the Base Register, the Index Register,
4408 // the Scale Value, and the Offset Value of the operand when queried.
4409 // COND_INTER causes an operand to generate six functions which return
4410 // the encoding code (ie - encoding bits for the instruction)
4411 // associated with each basic boolean condition for a conditional
4412 // instruction.
4413 //
4414 // Instructions specify two basic values for encoding. Again, a
4415 // function is available to check if the constant displacement is an
4416 // oop. They use the ins_encode keyword to specify their encoding
4417 // classes (which must be a sequence of enc_class names, and their
4418 // parameters, specified in the encoding block), and they use the
4419 // opcode keyword to specify, in order, their primary, secondary, and
4420 // tertiary opcode. Only the opcode sections which a particular
4421 // instruction needs for encoding need to be specified.
4422 encode %{
4423 enc_class cdql_enc(no_rax_rdx_RegI div)
4424 %{
4425 // Full implementation of Java idiv and irem; checks for
4426 // special case as described in JVM spec., p.243 & p.271.
4427 //
4428 // normal case special case
4429 //
4430 // input : rax: dividend min_int
4431 // reg: divisor -1
4432 //
4433 // output: rax: quotient (= rax idiv reg) min_int
4434 // rdx: remainder (= rax irem reg) 0
4435 //
4436 // Code sequnce:
4437 //
4438 // 0: 3d 00 00 00 80 cmp $0x80000000,%eax
4439 // 5: 75 07/08 jne e <normal>
4440 // 7: 33 d2 xor %edx,%edx
4441 // [div >= 8 -> offset + 1]
4442 // [REX_B]
4443 // 9: 83 f9 ff cmp $0xffffffffffffffff,$div
4444 // c: 74 03/04 je 11 <done>
4445 // 000000000000000e <normal>:
4446 // e: 99 cltd
4447 // [div >= 8 -> offset + 1]
4448 // [REX_B]
4449 // f: f7 f9 idiv $div
4450 // 0000000000000011 <done>:
4451 Label normal;
4452 Label done;
4453
4454 // cmp $0x80000000,%eax
4455 __ cmpl(as_Register(RAX_enc), 0x80000000);
4456
4457 // jne e <normal>
4458 __ jccb(Assembler::notEqual, normal);
4459
4460 // xor %edx,%edx
4461 __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
4462
4463 // cmp $0xffffffffffffffff,%ecx
4464 __ cmpl($div$$Register, -1);
4465
4466 // je 11 <done>
4467 __ jccb(Assembler::equal, done);
4468
4469 // <normal>
4470 // cltd
4471 __ bind(normal);
4472 __ cdql();
4473
4474 // idivl
4475 // <done>
4476 __ idivl($div$$Register);
4477 __ bind(done);
4478 %}
4479
4480 enc_class cdqq_enc(no_rax_rdx_RegL div)
4481 %{
4482 // Full implementation of Java ldiv and lrem; checks for
4483 // special case as described in JVM spec., p.243 & p.271.
4484 //
4485 // normal case special case
4486 //
4487 // input : rax: dividend min_long
4488 // reg: divisor -1
4489 //
4490 // output: rax: quotient (= rax idiv reg) min_long
4491 // rdx: remainder (= rax irem reg) 0
4492 //
4493 // Code sequnce:
4494 //
4495 // 0: 48 ba 00 00 00 00 00 mov $0x8000000000000000,%rdx
4496 // 7: 00 00 80
4497 // a: 48 39 d0 cmp %rdx,%rax
4498 // d: 75 08 jne 17 <normal>
4499 // f: 33 d2 xor %edx,%edx
4500 // 11: 48 83 f9 ff cmp $0xffffffffffffffff,$div
4501 // 15: 74 05 je 1c <done>
4502 // 0000000000000017 <normal>:
4503 // 17: 48 99 cqto
4504 // 19: 48 f7 f9 idiv $div
4505 // 000000000000001c <done>:
4506 Label normal;
4507 Label done;
4508
4509 // mov $0x8000000000000000,%rdx
4510 __ mov64(as_Register(RDX_enc), 0x8000000000000000);
4511
4512 // cmp %rdx,%rax
4513 __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
4514
4515 // jne 17 <normal>
4516 __ jccb(Assembler::notEqual, normal);
4517
4518 // xor %edx,%edx
4519 __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
4520
4521 // cmp $0xffffffffffffffff,$div
4522 __ cmpq($div$$Register, -1);
4523
4524 // je 1e <done>
4525 __ jccb(Assembler::equal, done);
4526
4527 // <normal>
4528 // cqto
4529 __ bind(normal);
4530 __ cdqq();
4531
4532 // idivq (note: must be emitted by the user of this rule)
4533 // <done>
4534 __ idivq($div$$Register);
4535 __ bind(done);
4536 %}
4537
4538 enc_class clear_avx %{
4539 DEBUG_ONLY(int off0 = __ offset());
4540 if (generate_vzeroupper(Compile::current())) {
4541 // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
4542 // Clear upper bits of YMM registers when current compiled code uses
4543 // wide vectors to avoid AVX <-> SSE transition penalty during call.
4544 __ vzeroupper();
4545 }
4546 DEBUG_ONLY(int off1 = __ offset());
4547 assert(off1 - off0 == clear_avx_size(), "correct size prediction");
4548 %}
4549
4550 enc_class Java_To_Runtime(method meth) %{
4551 __ lea(r10, RuntimeAddress((address)$meth$$method));
4552 __ call(r10);
4553 __ post_call_nop();
4554 %}
4555
4556 enc_class Java_Static_Call(method meth)
4557 %{
4558 // JAVA STATIC CALL
4559 // CALL to fixup routine. Fixup routine uses ScopeDesc info to
4560 // determine who we intended to call.
4561 if (!_method) {
4562 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
4563 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
4564 // The NOP here is purely to ensure that eliding a call to
4565 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
4566 __ nop(5);
4567 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
4568 } else {
4569 int method_index = resolved_method_index(masm);
4570 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
4571 : static_call_Relocation::spec(method_index);
4572 address mark = __ pc();
4573 int call_offset = __ offset();
4574 __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
4575 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
4576 // Calls of the same statically bound method can share
4577 // a stub to the interpreter.
4578 __ code()->shared_stub_to_interp_for(_method, call_offset);
4579 } else {
4580 // Emit stubs for static call.
4581 address stub = CompiledDirectCall::emit_to_interp_stub(masm, mark);
4582 __ clear_inst_mark();
4583 if (stub == nullptr) {
4584 ciEnv::current()->record_failure("CodeCache is full");
4585 return;
4586 }
4587 }
4588 }
4589 __ post_call_nop();
4590 %}
4591
4592 enc_class Java_Dynamic_Call(method meth) %{
4593 __ ic_call((address)$meth$$method, resolved_method_index(masm));
4594 __ post_call_nop();
4595 %}
4596
4597 enc_class call_epilog %{
4598 if (VerifyStackAtCalls) {
4599 // Check that stack depth is unchanged: find majik cookie on stack
4600 int framesize = ra_->reg2offset_unchecked(OptoReg::add(ra_->_matcher._old_SP, -3*VMRegImpl::slots_per_word));
4601 Label L;
4602 __ cmpptr(Address(rsp, framesize), (int32_t)0xbadb100d);
4603 __ jccb(Assembler::equal, L);
4604 // Die if stack mismatch
4605 __ int3();
4606 __ bind(L);
4607 }
4608 %}
4609
4610 %}
4611
4612 //----------FRAME--------------------------------------------------------------
4613 // Definition of frame structure and management information.
4614 //
4615 // S T A C K L A Y O U T Allocators stack-slot number
4616 // | (to get allocators register number
4617 // G Owned by | | v add OptoReg::stack0())
4618 // r CALLER | |
4619 // o | +--------+ pad to even-align allocators stack-slot
4620 // w V | pad0 | numbers; owned by CALLER
4621 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
4622 // h ^ | in | 5
4623 // | | args | 4 Holes in incoming args owned by SELF
4624 // | | | | 3
4625 // | | +--------+
4626 // V | | old out| Empty on Intel, window on Sparc
4627 // | old |preserve| Must be even aligned.
4628 // | SP-+--------+----> Matcher::_old_SP, even aligned
4629 // | | in | 3 area for Intel ret address
4630 // Owned by |preserve| Empty on Sparc.
4631 // SELF +--------+
4632 // | | pad2 | 2 pad to align old SP
4633 // | +--------+ 1
4634 // | | locks | 0
4635 // | +--------+----> OptoReg::stack0(), even aligned
4636 // | | pad1 | 11 pad to align new SP
4637 // | +--------+
4638 // | | | 10
4639 // | | spills | 9 spills
4640 // V | | 8 (pad0 slot for callee)
4641 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
4642 // ^ | out | 7
4643 // | | args | 6 Holes in outgoing args owned by CALLEE
4644 // Owned by +--------+
4645 // CALLEE | new out| 6 Empty on Intel, window on Sparc
4646 // | new |preserve| Must be even-aligned.
4647 // | SP-+--------+----> Matcher::_new_SP, even aligned
4648 // | | |
4649 //
4650 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
4651 // known from SELF's arguments and the Java calling convention.
4652 // Region 6-7 is determined per call site.
4653 // Note 2: If the calling convention leaves holes in the incoming argument
4654 // area, those holes are owned by SELF. Holes in the outgoing area
4655 // are owned by the CALLEE. Holes should not be necessary in the
4656 // incoming area, as the Java calling convention is completely under
4657 // the control of the AD file. Doubles can be sorted and packed to
4658 // avoid holes. Holes in the outgoing arguments may be necessary for
4659 // varargs C calling conventions.
4660 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
4661 // even aligned with pad0 as needed.
4662 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
4663 // region 6-11 is even aligned; it may be padded out more so that
4664 // the region from SP to FP meets the minimum stack alignment.
4665 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
4666 // alignment. Region 11, pad1, may be dynamically extended so that
4667 // SP meets the minimum alignment.
4668
4669 frame
4670 %{
4671 // These three registers define part of the calling convention
4672 // between compiled code and the interpreter.
4673 inline_cache_reg(RAX); // Inline Cache Register
4674
4675 // Optional: name the operand used by cisc-spilling to access
4676 // [stack_pointer + offset]
4677 cisc_spilling_operand_name(indOffset32);
4678
4679 // Number of stack slots consumed by locking an object
4680 sync_stack_slots(2);
4681
4682 // Compiled code's Frame Pointer
4683 frame_pointer(RSP);
4684
4685 // Stack alignment requirement
4686 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
4687
4688 // Number of outgoing stack slots killed above the out_preserve_stack_slots
4689 // for calls to C. Supports the var-args backing area for register parms.
4690 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
4691
4692 // The after-PROLOG location of the return address. Location of
4693 // return address specifies a type (REG or STACK) and a number
4694 // representing the register number (i.e. - use a register name) or
4695 // stack slot.
4696 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4697 // Otherwise, it is above the locks and verification slot and alignment word
4698 return_addr(STACK - 2 +
4699 align_up((Compile::current()->in_preserve_stack_slots() +
4700 Compile::current()->fixed_slots()),
4701 stack_alignment_in_slots()));
4702
4703 // Location of compiled Java return values. Same as C for now.
4704 return_value
4705 %{
4706 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4707 "only return normal values");
4708
4709 static const int lo[Op_RegL + 1] = {
4710 0,
4711 0,
4712 RAX_num, // Op_RegN
4713 RAX_num, // Op_RegI
4714 RAX_num, // Op_RegP
4715 XMM0_num, // Op_RegF
4716 XMM0_num, // Op_RegD
4717 RAX_num // Op_RegL
4718 };
4719 static const int hi[Op_RegL + 1] = {
4720 0,
4721 0,
4722 OptoReg::Bad, // Op_RegN
4723 OptoReg::Bad, // Op_RegI
4724 RAX_H_num, // Op_RegP
4725 OptoReg::Bad, // Op_RegF
4726 XMM0b_num, // Op_RegD
4727 RAX_H_num // Op_RegL
4728 };
4729 // Excluded flags and vector registers.
4730 assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
4731 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4732 %}
4733 %}
4734
4735 //----------ATTRIBUTES---------------------------------------------------------
4736 //----------Operand Attributes-------------------------------------------------
4737 op_attrib op_cost(0); // Required cost attribute
4738
4739 //----------Instruction Attributes---------------------------------------------
4740 ins_attrib ins_cost(100); // Required cost attribute
4741 ins_attrib ins_size(8); // Required size attribute (in bits)
4742 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4743 // a non-matching short branch variant
4744 // of some long branch?
4745 ins_attrib ins_alignment(1); // Required alignment attribute (must
4746 // be a power of 2) specifies the
4747 // alignment that some part of the
4748 // instruction (not necessarily the
4749 // start) requires. If > 1, a
4750 // compute_padding() function must be
4751 // provided for the instruction
4752
4753 // Whether this node is expanded during code emission into a sequence of
4754 // instructions and the first instruction can perform an implicit null check.
4755 ins_attrib ins_is_late_expanded_null_check_candidate(false);
4756
4757 //----------OPERANDS-----------------------------------------------------------
4758 // Operand definitions must precede instruction definitions for correct parsing
4759 // in the ADLC because operands constitute user defined types which are used in
4760 // instruction definitions.
4761
4762 //----------Simple Operands----------------------------------------------------
4763 // Immediate Operands
4764 // Integer Immediate
4765 operand immI()
4766 %{
4767 match(ConI);
4768
4769 op_cost(10);
4770 format %{ %}
4771 interface(CONST_INTER);
4772 %}
4773
4774 // Constant for test vs zero
4775 operand immI_0()
4776 %{
4777 predicate(n->get_int() == 0);
4778 match(ConI);
4779
4780 op_cost(0);
4781 format %{ %}
4782 interface(CONST_INTER);
4783 %}
4784
4785 // Constant for increment
4786 operand immI_1()
4787 %{
4788 predicate(n->get_int() == 1);
4789 match(ConI);
4790
4791 op_cost(0);
4792 format %{ %}
4793 interface(CONST_INTER);
4794 %}
4795
4796 // Constant for decrement
4797 operand immI_M1()
4798 %{
4799 predicate(n->get_int() == -1);
4800 match(ConI);
4801
4802 op_cost(0);
4803 format %{ %}
4804 interface(CONST_INTER);
4805 %}
4806
4807 operand immI_2()
4808 %{
4809 predicate(n->get_int() == 2);
4810 match(ConI);
4811
4812 op_cost(0);
4813 format %{ %}
4814 interface(CONST_INTER);
4815 %}
4816
4817 operand immI_4()
4818 %{
4819 predicate(n->get_int() == 4);
4820 match(ConI);
4821
4822 op_cost(0);
4823 format %{ %}
4824 interface(CONST_INTER);
4825 %}
4826
4827 operand immI_8()
4828 %{
4829 predicate(n->get_int() == 8);
4830 match(ConI);
4831
4832 op_cost(0);
4833 format %{ %}
4834 interface(CONST_INTER);
4835 %}
4836
4837 // Valid scale values for addressing modes
4838 operand immI2()
4839 %{
4840 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4841 match(ConI);
4842
4843 format %{ %}
4844 interface(CONST_INTER);
4845 %}
4846
4847 operand immU7()
4848 %{
4849 predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
4850 match(ConI);
4851
4852 op_cost(5);
4853 format %{ %}
4854 interface(CONST_INTER);
4855 %}
4856
4857 operand immI8()
4858 %{
4859 predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
4860 match(ConI);
4861
4862 op_cost(5);
4863 format %{ %}
4864 interface(CONST_INTER);
4865 %}
4866
4867 operand immU8()
4868 %{
4869 predicate((0 <= n->get_int()) && (n->get_int() <= 255));
4870 match(ConI);
4871
4872 op_cost(5);
4873 format %{ %}
4874 interface(CONST_INTER);
4875 %}
4876
4877 operand immI16()
4878 %{
4879 predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
4880 match(ConI);
4881
4882 op_cost(10);
4883 format %{ %}
4884 interface(CONST_INTER);
4885 %}
4886
4887 // Int Immediate non-negative
4888 operand immU31()
4889 %{
4890 predicate(n->get_int() >= 0);
4891 match(ConI);
4892
4893 op_cost(0);
4894 format %{ %}
4895 interface(CONST_INTER);
4896 %}
4897
4898 // Pointer Immediate
4899 operand immP()
4900 %{
4901 match(ConP);
4902
4903 op_cost(10);
4904 format %{ %}
4905 interface(CONST_INTER);
4906 %}
4907
4908 // Null Pointer Immediate
4909 operand immP0()
4910 %{
4911 predicate(n->get_ptr() == 0);
4912 match(ConP);
4913
4914 op_cost(5);
4915 format %{ %}
4916 interface(CONST_INTER);
4917 %}
4918
4919 // Pointer Immediate
4920 operand immN() %{
4921 match(ConN);
4922
4923 op_cost(10);
4924 format %{ %}
4925 interface(CONST_INTER);
4926 %}
4927
4928 operand immNKlass() %{
4929 match(ConNKlass);
4930
4931 op_cost(10);
4932 format %{ %}
4933 interface(CONST_INTER);
4934 %}
4935
4936 // Null Pointer Immediate
4937 operand immN0() %{
4938 predicate(n->get_narrowcon() == 0);
4939 match(ConN);
4940
4941 op_cost(5);
4942 format %{ %}
4943 interface(CONST_INTER);
4944 %}
4945
4946 operand immP31()
4947 %{
4948 predicate(n->as_Type()->type()->is_ptr()->reloc() == relocInfo::none
4949 && (n->get_ptr() >> 31) == 0);
4950 match(ConP);
4951
4952 op_cost(5);
4953 format %{ %}
4954 interface(CONST_INTER);
4955 %}
4956
4957
4958 // Long Immediate
4959 operand immL()
4960 %{
4961 match(ConL);
4962
4963 op_cost(20);
4964 format %{ %}
4965 interface(CONST_INTER);
4966 %}
4967
4968 // Long Immediate 8-bit
4969 operand immL8()
4970 %{
4971 predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
4972 match(ConL);
4973
4974 op_cost(5);
4975 format %{ %}
4976 interface(CONST_INTER);
4977 %}
4978
4979 // Long Immediate 32-bit unsigned
4980 operand immUL32()
4981 %{
4982 predicate(n->get_long() == (unsigned int) (n->get_long()));
4983 match(ConL);
4984
4985 op_cost(10);
4986 format %{ %}
4987 interface(CONST_INTER);
4988 %}
4989
4990 // Long Immediate 32-bit signed
4991 operand immL32()
4992 %{
4993 predicate(n->get_long() == (int) (n->get_long()));
4994 match(ConL);
4995
4996 op_cost(15);
4997 format %{ %}
4998 interface(CONST_INTER);
4999 %}
5000
5001 operand immL_Pow2()
5002 %{
5003 predicate(is_power_of_2((julong)n->get_long()));
5004 match(ConL);
5005
5006 op_cost(15);
5007 format %{ %}
5008 interface(CONST_INTER);
5009 %}
5010
5011 operand immL_NotPow2()
5012 %{
5013 predicate(is_power_of_2((julong)~n->get_long()));
5014 match(ConL);
5015
5016 op_cost(15);
5017 format %{ %}
5018 interface(CONST_INTER);
5019 %}
5020
5021 // Long Immediate zero
5022 operand immL0()
5023 %{
5024 predicate(n->get_long() == 0L);
5025 match(ConL);
5026
5027 op_cost(10);
5028 format %{ %}
5029 interface(CONST_INTER);
5030 %}
5031
5032 // Constant for increment
5033 operand immL1()
5034 %{
5035 predicate(n->get_long() == 1);
5036 match(ConL);
5037
5038 format %{ %}
5039 interface(CONST_INTER);
5040 %}
5041
5042 // Constant for decrement
5043 operand immL_M1()
5044 %{
5045 predicate(n->get_long() == -1);
5046 match(ConL);
5047
5048 format %{ %}
5049 interface(CONST_INTER);
5050 %}
5051
5052 // Long Immediate: low 32-bit mask
5053 operand immL_32bits()
5054 %{
5055 predicate(n->get_long() == 0xFFFFFFFFL);
5056 match(ConL);
5057 op_cost(20);
5058
5059 format %{ %}
5060 interface(CONST_INTER);
5061 %}
5062
5063 // Int Immediate: 2^n-1, positive
5064 operand immI_Pow2M1()
5065 %{
5066 predicate((n->get_int() > 0)
5067 && is_power_of_2((juint)n->get_int() + 1));
5068 match(ConI);
5069
5070 op_cost(20);
5071 format %{ %}
5072 interface(CONST_INTER);
5073 %}
5074
5075 // Float Immediate zero
5076 operand immF0()
5077 %{
5078 predicate(jint_cast(n->getf()) == 0);
5079 match(ConF);
5080
5081 op_cost(5);
5082 format %{ %}
5083 interface(CONST_INTER);
5084 %}
5085
5086 // Float Immediate
5087 operand immF()
5088 %{
5089 match(ConF);
5090
5091 op_cost(15);
5092 format %{ %}
5093 interface(CONST_INTER);
5094 %}
5095
5096 // Half Float Immediate
5097 operand immH()
5098 %{
5099 match(ConH);
5100
5101 op_cost(15);
5102 format %{ %}
5103 interface(CONST_INTER);
5104 %}
5105
5106 // Double Immediate zero
5107 operand immD0()
5108 %{
5109 predicate(jlong_cast(n->getd()) == 0);
5110 match(ConD);
5111
5112 op_cost(5);
5113 format %{ %}
5114 interface(CONST_INTER);
5115 %}
5116
5117 // Double Immediate
5118 operand immD()
5119 %{
5120 match(ConD);
5121
5122 op_cost(15);
5123 format %{ %}
5124 interface(CONST_INTER);
5125 %}
5126
5127 // Immediates for special shifts (sign extend)
5128
5129 // Constants for increment
5130 operand immI_16()
5131 %{
5132 predicate(n->get_int() == 16);
5133 match(ConI);
5134
5135 format %{ %}
5136 interface(CONST_INTER);
5137 %}
5138
5139 operand immI_24()
5140 %{
5141 predicate(n->get_int() == 24);
5142 match(ConI);
5143
5144 format %{ %}
5145 interface(CONST_INTER);
5146 %}
5147
5148 // Constant for byte-wide masking
5149 operand immI_255()
5150 %{
5151 predicate(n->get_int() == 255);
5152 match(ConI);
5153
5154 format %{ %}
5155 interface(CONST_INTER);
5156 %}
5157
5158 // Constant for short-wide masking
5159 operand immI_65535()
5160 %{
5161 predicate(n->get_int() == 65535);
5162 match(ConI);
5163
5164 format %{ %}
5165 interface(CONST_INTER);
5166 %}
5167
5168 // Constant for byte-wide masking
5169 operand immL_255()
5170 %{
5171 predicate(n->get_long() == 255);
5172 match(ConL);
5173
5174 format %{ %}
5175 interface(CONST_INTER);
5176 %}
5177
5178 // Constant for short-wide masking
5179 operand immL_65535()
5180 %{
5181 predicate(n->get_long() == 65535);
5182 match(ConL);
5183
5184 format %{ %}
5185 interface(CONST_INTER);
5186 %}
5187
5188 // AOT Runtime Constants Address
5189 operand immAOTRuntimeConstantsAddress()
5190 %{
5191 // Check if the address is in the range of AOT Runtime Constants
5192 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
5193 match(ConP);
5194
5195 op_cost(0);
5196 format %{ %}
5197 interface(CONST_INTER);
5198 %}
5199
5200 operand kReg()
5201 %{
5202 constraint(ALLOC_IN_RC(vectmask_reg));
5203 match(RegVectMask);
5204 format %{%}
5205 interface(REG_INTER);
5206 %}
5207
5208 // Register Operands
5209 // Integer Register
5210 operand rRegI()
5211 %{
5212 constraint(ALLOC_IN_RC(int_reg));
5213 match(RegI);
5214
5215 match(rax_RegI);
5216 match(rbx_RegI);
5217 match(rcx_RegI);
5218 match(rdx_RegI);
5219 match(rdi_RegI);
5220
5221 format %{ %}
5222 interface(REG_INTER);
5223 %}
5224
5225 // Special Registers
5226 operand rax_RegI()
5227 %{
5228 constraint(ALLOC_IN_RC(int_rax_reg));
5229 match(RegI);
5230 match(rRegI);
5231
5232 format %{ "RAX" %}
5233 interface(REG_INTER);
5234 %}
5235
5236 // Special Registers
5237 operand rbx_RegI()
5238 %{
5239 constraint(ALLOC_IN_RC(int_rbx_reg));
5240 match(RegI);
5241 match(rRegI);
5242
5243 format %{ "RBX" %}
5244 interface(REG_INTER);
5245 %}
5246
5247 operand rcx_RegI()
5248 %{
5249 constraint(ALLOC_IN_RC(int_rcx_reg));
5250 match(RegI);
5251 match(rRegI);
5252
5253 format %{ "RCX" %}
5254 interface(REG_INTER);
5255 %}
5256
5257 operand rdx_RegI()
5258 %{
5259 constraint(ALLOC_IN_RC(int_rdx_reg));
5260 match(RegI);
5261 match(rRegI);
5262
5263 format %{ "RDX" %}
5264 interface(REG_INTER);
5265 %}
5266
5267 operand rdi_RegI()
5268 %{
5269 constraint(ALLOC_IN_RC(int_rdi_reg));
5270 match(RegI);
5271 match(rRegI);
5272
5273 format %{ "RDI" %}
5274 interface(REG_INTER);
5275 %}
5276
5277 operand no_rax_rdx_RegI()
5278 %{
5279 constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
5280 match(RegI);
5281 match(rbx_RegI);
5282 match(rcx_RegI);
5283 match(rdi_RegI);
5284
5285 format %{ %}
5286 interface(REG_INTER);
5287 %}
5288
5289 operand no_rbp_r13_RegI()
5290 %{
5291 constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
5292 match(RegI);
5293 match(rRegI);
5294 match(rax_RegI);
5295 match(rbx_RegI);
5296 match(rcx_RegI);
5297 match(rdx_RegI);
5298 match(rdi_RegI);
5299
5300 format %{ %}
5301 interface(REG_INTER);
5302 %}
5303
5304 // Pointer Register
5305 operand any_RegP()
5306 %{
5307 constraint(ALLOC_IN_RC(any_reg));
5308 match(RegP);
5309 match(rax_RegP);
5310 match(rbx_RegP);
5311 match(rdi_RegP);
5312 match(rsi_RegP);
5313 match(rbp_RegP);
5314 match(r15_RegP);
5315 match(rRegP);
5316
5317 format %{ %}
5318 interface(REG_INTER);
5319 %}
5320
5321 operand rRegP()
5322 %{
5323 constraint(ALLOC_IN_RC(ptr_reg));
5324 match(RegP);
5325 match(rax_RegP);
5326 match(rbx_RegP);
5327 match(rdi_RegP);
5328 match(rsi_RegP);
5329 match(rbp_RegP); // See Q&A below about
5330 match(r15_RegP); // r15_RegP and rbp_RegP.
5331
5332 format %{ %}
5333 interface(REG_INTER);
5334 %}
5335
5336 operand rRegN() %{
5337 constraint(ALLOC_IN_RC(int_reg));
5338 match(RegN);
5339
5340 format %{ %}
5341 interface(REG_INTER);
5342 %}
5343
5344 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
5345 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
5346 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
5347 // The output of an instruction is controlled by the allocator, which respects
5348 // register class masks, not match rules. Unless an instruction mentions
5349 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
5350 // by the allocator as an input.
5351 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
5352 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
5353 // result, RBP is not included in the output of the instruction either.
5354
5355 // This operand is not allowed to use RBP even if
5356 // RBP is not used to hold the frame pointer.
5357 operand no_rbp_RegP()
5358 %{
5359 constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
5360 match(RegP);
5361 match(rbx_RegP);
5362 match(rsi_RegP);
5363 match(rdi_RegP);
5364
5365 format %{ %}
5366 interface(REG_INTER);
5367 %}
5368
5369 // Special Registers
5370 // Return a pointer value
5371 operand rax_RegP()
5372 %{
5373 constraint(ALLOC_IN_RC(ptr_rax_reg));
5374 match(RegP);
5375 match(rRegP);
5376
5377 format %{ %}
5378 interface(REG_INTER);
5379 %}
5380
5381 // Special Registers
5382 // Return a compressed pointer value
5383 operand rax_RegN()
5384 %{
5385 constraint(ALLOC_IN_RC(int_rax_reg));
5386 match(RegN);
5387 match(rRegN);
5388
5389 format %{ %}
5390 interface(REG_INTER);
5391 %}
5392
5393 // Used in AtomicAdd
5394 operand rbx_RegP()
5395 %{
5396 constraint(ALLOC_IN_RC(ptr_rbx_reg));
5397 match(RegP);
5398 match(rRegP);
5399
5400 format %{ %}
5401 interface(REG_INTER);
5402 %}
5403
5404 operand rsi_RegP()
5405 %{
5406 constraint(ALLOC_IN_RC(ptr_rsi_reg));
5407 match(RegP);
5408 match(rRegP);
5409
5410 format %{ %}
5411 interface(REG_INTER);
5412 %}
5413
5414 operand rbp_RegP()
5415 %{
5416 constraint(ALLOC_IN_RC(ptr_rbp_reg));
5417 match(RegP);
5418 match(rRegP);
5419
5420 format %{ %}
5421 interface(REG_INTER);
5422 %}
5423
5424 // Used in rep stosq
5425 operand rdi_RegP()
5426 %{
5427 constraint(ALLOC_IN_RC(ptr_rdi_reg));
5428 match(RegP);
5429 match(rRegP);
5430
5431 format %{ %}
5432 interface(REG_INTER);
5433 %}
5434
5435 operand r15_RegP()
5436 %{
5437 constraint(ALLOC_IN_RC(ptr_r15_reg));
5438 match(RegP);
5439 match(rRegP);
5440
5441 format %{ %}
5442 interface(REG_INTER);
5443 %}
5444
5445 operand rRegL()
5446 %{
5447 constraint(ALLOC_IN_RC(long_reg));
5448 match(RegL);
5449 match(rax_RegL);
5450 match(rdx_RegL);
5451
5452 format %{ %}
5453 interface(REG_INTER);
5454 %}
5455
5456 // Special Registers
5457 operand no_rax_rdx_RegL()
5458 %{
5459 constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
5460 match(RegL);
5461 match(rRegL);
5462
5463 format %{ %}
5464 interface(REG_INTER);
5465 %}
5466
5467 operand rax_RegL()
5468 %{
5469 constraint(ALLOC_IN_RC(long_rax_reg));
5470 match(RegL);
5471 match(rRegL);
5472
5473 format %{ "RAX" %}
5474 interface(REG_INTER);
5475 %}
5476
5477 operand rcx_RegL()
5478 %{
5479 constraint(ALLOC_IN_RC(long_rcx_reg));
5480 match(RegL);
5481 match(rRegL);
5482
5483 format %{ %}
5484 interface(REG_INTER);
5485 %}
5486
5487 operand rdx_RegL()
5488 %{
5489 constraint(ALLOC_IN_RC(long_rdx_reg));
5490 match(RegL);
5491 match(rRegL);
5492
5493 format %{ %}
5494 interface(REG_INTER);
5495 %}
5496
5497 operand r11_RegL()
5498 %{
5499 constraint(ALLOC_IN_RC(long_r11_reg));
5500 match(RegL);
5501 match(rRegL);
5502
5503 format %{ %}
5504 interface(REG_INTER);
5505 %}
5506
5507 operand no_rbp_r13_RegL()
5508 %{
5509 constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
5510 match(RegL);
5511 match(rRegL);
5512 match(rax_RegL);
5513 match(rcx_RegL);
5514 match(rdx_RegL);
5515
5516 format %{ %}
5517 interface(REG_INTER);
5518 %}
5519
5520 // Flags register, used as output of compare instructions
5521 operand rFlagsReg()
5522 %{
5523 constraint(ALLOC_IN_RC(int_flags));
5524 match(RegFlags);
5525
5526 format %{ "RFLAGS" %}
5527 interface(REG_INTER);
5528 %}
5529
5530 // Flags register, used as output of FLOATING POINT compare instructions
5531 operand rFlagsRegU()
5532 %{
5533 constraint(ALLOC_IN_RC(int_flags));
5534 match(RegFlags);
5535
5536 format %{ "RFLAGS_U" %}
5537 interface(REG_INTER);
5538 %}
5539
5540 operand rFlagsRegUCF() %{
5541 constraint(ALLOC_IN_RC(int_flags));
5542 match(RegFlags);
5543 predicate(!UseAPX || !VM_Version::supports_avx10_2());
5544
5545 format %{ "RFLAGS_U_CF" %}
5546 interface(REG_INTER);
5547 %}
5548
5549 operand rFlagsRegUCFE() %{
5550 constraint(ALLOC_IN_RC(int_flags));
5551 match(RegFlags);
5552 predicate(UseAPX && VM_Version::supports_avx10_2());
5553
5554 format %{ "RFLAGS_U_CFE" %}
5555 interface(REG_INTER);
5556 %}
5557
5558 // Float register operands
5559 operand regF() %{
5560 constraint(ALLOC_IN_RC(float_reg));
5561 match(RegF);
5562
5563 format %{ %}
5564 interface(REG_INTER);
5565 %}
5566
5567 // Float register operands
5568 operand legRegF() %{
5569 constraint(ALLOC_IN_RC(float_reg_legacy));
5570 match(RegF);
5571
5572 format %{ %}
5573 interface(REG_INTER);
5574 %}
5575
5576 // Float register operands
5577 operand vlRegF() %{
5578 constraint(ALLOC_IN_RC(float_reg_vl));
5579 match(RegF);
5580
5581 format %{ %}
5582 interface(REG_INTER);
5583 %}
5584
5585 // Double register operands
5586 operand regD() %{
5587 constraint(ALLOC_IN_RC(double_reg));
5588 match(RegD);
5589
5590 format %{ %}
5591 interface(REG_INTER);
5592 %}
5593
5594 // Double register operands
5595 operand legRegD() %{
5596 constraint(ALLOC_IN_RC(double_reg_legacy));
5597 match(RegD);
5598
5599 format %{ %}
5600 interface(REG_INTER);
5601 %}
5602
5603 // Double register operands
5604 operand vlRegD() %{
5605 constraint(ALLOC_IN_RC(double_reg_vl));
5606 match(RegD);
5607
5608 format %{ %}
5609 interface(REG_INTER);
5610 %}
5611
5612 //----------Memory Operands----------------------------------------------------
5613 // Direct Memory Operand
5614 // operand direct(immP addr)
5615 // %{
5616 // match(addr);
5617
5618 // format %{ "[$addr]" %}
5619 // interface(MEMORY_INTER) %{
5620 // base(0xFFFFFFFF);
5621 // index(0x4);
5622 // scale(0x0);
5623 // disp($addr);
5624 // %}
5625 // %}
5626
5627 // Indirect Memory Operand
5628 operand indirect(any_RegP reg)
5629 %{
5630 constraint(ALLOC_IN_RC(ptr_reg));
5631 match(reg);
5632
5633 format %{ "[$reg]" %}
5634 interface(MEMORY_INTER) %{
5635 base($reg);
5636 index(0x4);
5637 scale(0x0);
5638 disp(0x0);
5639 %}
5640 %}
5641
5642 // Indirect Memory Plus Short Offset Operand
5643 operand indOffset8(any_RegP reg, immL8 off)
5644 %{
5645 constraint(ALLOC_IN_RC(ptr_reg));
5646 match(AddP reg off);
5647
5648 format %{ "[$reg + $off (8-bit)]" %}
5649 interface(MEMORY_INTER) %{
5650 base($reg);
5651 index(0x4);
5652 scale(0x0);
5653 disp($off);
5654 %}
5655 %}
5656
5657 // Indirect Memory Plus Long Offset Operand
5658 operand indOffset32(any_RegP reg, immL32 off)
5659 %{
5660 constraint(ALLOC_IN_RC(ptr_reg));
5661 match(AddP reg off);
5662
5663 format %{ "[$reg + $off (32-bit)]" %}
5664 interface(MEMORY_INTER) %{
5665 base($reg);
5666 index(0x4);
5667 scale(0x0);
5668 disp($off);
5669 %}
5670 %}
5671
5672 // Indirect Memory Plus Index Register Plus Offset Operand
5673 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
5674 %{
5675 constraint(ALLOC_IN_RC(ptr_reg));
5676 match(AddP (AddP reg lreg) off);
5677
5678 op_cost(10);
5679 format %{"[$reg + $off + $lreg]" %}
5680 interface(MEMORY_INTER) %{
5681 base($reg);
5682 index($lreg);
5683 scale(0x0);
5684 disp($off);
5685 %}
5686 %}
5687
5688 // Indirect Memory Plus Index Register Plus Offset Operand
5689 operand indIndex(any_RegP reg, rRegL lreg)
5690 %{
5691 constraint(ALLOC_IN_RC(ptr_reg));
5692 match(AddP reg lreg);
5693
5694 op_cost(10);
5695 format %{"[$reg + $lreg]" %}
5696 interface(MEMORY_INTER) %{
5697 base($reg);
5698 index($lreg);
5699 scale(0x0);
5700 disp(0x0);
5701 %}
5702 %}
5703
5704 // Indirect Memory Times Scale Plus Index Register
5705 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
5706 %{
5707 constraint(ALLOC_IN_RC(ptr_reg));
5708 match(AddP reg (LShiftL lreg scale));
5709
5710 op_cost(10);
5711 format %{"[$reg + $lreg << $scale]" %}
5712 interface(MEMORY_INTER) %{
5713 base($reg);
5714 index($lreg);
5715 scale($scale);
5716 disp(0x0);
5717 %}
5718 %}
5719
5720 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
5721 %{
5722 constraint(ALLOC_IN_RC(ptr_reg));
5723 predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
5724 match(AddP reg (LShiftL (ConvI2L idx) scale));
5725
5726 op_cost(10);
5727 format %{"[$reg + pos $idx << $scale]" %}
5728 interface(MEMORY_INTER) %{
5729 base($reg);
5730 index($idx);
5731 scale($scale);
5732 disp(0x0);
5733 %}
5734 %}
5735
5736 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
5737 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
5738 %{
5739 constraint(ALLOC_IN_RC(ptr_reg));
5740 match(AddP (AddP reg (LShiftL lreg scale)) off);
5741
5742 op_cost(10);
5743 format %{"[$reg + $off + $lreg << $scale]" %}
5744 interface(MEMORY_INTER) %{
5745 base($reg);
5746 index($lreg);
5747 scale($scale);
5748 disp($off);
5749 %}
5750 %}
5751
5752 // Indirect Memory Plus Positive Index Register Plus Offset Operand
5753 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
5754 %{
5755 constraint(ALLOC_IN_RC(ptr_reg));
5756 predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
5757 match(AddP (AddP reg (ConvI2L idx)) off);
5758
5759 op_cost(10);
5760 format %{"[$reg + $off + $idx]" %}
5761 interface(MEMORY_INTER) %{
5762 base($reg);
5763 index($idx);
5764 scale(0x0);
5765 disp($off);
5766 %}
5767 %}
5768
5769 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
5770 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
5771 %{
5772 constraint(ALLOC_IN_RC(ptr_reg));
5773 predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
5774 match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
5775
5776 op_cost(10);
5777 format %{"[$reg + $off + $idx << $scale]" %}
5778 interface(MEMORY_INTER) %{
5779 base($reg);
5780 index($idx);
5781 scale($scale);
5782 disp($off);
5783 %}
5784 %}
5785
5786 // Indirect Narrow Oop Plus Offset Operand
5787 // Note: x86 architecture doesn't support "scale * index + offset" without a base
5788 // we can't free r12 even with CompressedOops::base() == nullptr.
5789 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
5790 predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
5791 constraint(ALLOC_IN_RC(ptr_reg));
5792 match(AddP (DecodeN reg) off);
5793
5794 op_cost(10);
5795 format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
5796 interface(MEMORY_INTER) %{
5797 base(0xc); // R12
5798 index($reg);
5799 scale(0x3);
5800 disp($off);
5801 %}
5802 %}
5803
5804 // Indirect Memory Operand
5805 operand indirectNarrow(rRegN reg)
5806 %{
5807 predicate(CompressedOops::shift() == 0);
5808 constraint(ALLOC_IN_RC(ptr_reg));
5809 match(DecodeN reg);
5810
5811 format %{ "[$reg]" %}
5812 interface(MEMORY_INTER) %{
5813 base($reg);
5814 index(0x4);
5815 scale(0x0);
5816 disp(0x0);
5817 %}
5818 %}
5819
5820 // Indirect Memory Plus Short Offset Operand
5821 operand indOffset8Narrow(rRegN reg, immL8 off)
5822 %{
5823 predicate(CompressedOops::shift() == 0);
5824 constraint(ALLOC_IN_RC(ptr_reg));
5825 match(AddP (DecodeN reg) off);
5826
5827 format %{ "[$reg + $off (8-bit)]" %}
5828 interface(MEMORY_INTER) %{
5829 base($reg);
5830 index(0x4);
5831 scale(0x0);
5832 disp($off);
5833 %}
5834 %}
5835
5836 // Indirect Memory Plus Long Offset Operand
5837 operand indOffset32Narrow(rRegN reg, immL32 off)
5838 %{
5839 predicate(CompressedOops::shift() == 0);
5840 constraint(ALLOC_IN_RC(ptr_reg));
5841 match(AddP (DecodeN reg) off);
5842
5843 format %{ "[$reg + $off (32-bit)]" %}
5844 interface(MEMORY_INTER) %{
5845 base($reg);
5846 index(0x4);
5847 scale(0x0);
5848 disp($off);
5849 %}
5850 %}
5851
5852 // Indirect Memory Plus Index Register Plus Offset Operand
5853 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
5854 %{
5855 predicate(CompressedOops::shift() == 0);
5856 constraint(ALLOC_IN_RC(ptr_reg));
5857 match(AddP (AddP (DecodeN reg) lreg) off);
5858
5859 op_cost(10);
5860 format %{"[$reg + $off + $lreg]" %}
5861 interface(MEMORY_INTER) %{
5862 base($reg);
5863 index($lreg);
5864 scale(0x0);
5865 disp($off);
5866 %}
5867 %}
5868
5869 // Indirect Memory Plus Index Register Plus Offset Operand
5870 operand indIndexNarrow(rRegN reg, rRegL lreg)
5871 %{
5872 predicate(CompressedOops::shift() == 0);
5873 constraint(ALLOC_IN_RC(ptr_reg));
5874 match(AddP (DecodeN reg) lreg);
5875
5876 op_cost(10);
5877 format %{"[$reg + $lreg]" %}
5878 interface(MEMORY_INTER) %{
5879 base($reg);
5880 index($lreg);
5881 scale(0x0);
5882 disp(0x0);
5883 %}
5884 %}
5885
5886 // Indirect Memory Times Scale Plus Index Register
5887 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
5888 %{
5889 predicate(CompressedOops::shift() == 0);
5890 constraint(ALLOC_IN_RC(ptr_reg));
5891 match(AddP (DecodeN reg) (LShiftL lreg scale));
5892
5893 op_cost(10);
5894 format %{"[$reg + $lreg << $scale]" %}
5895 interface(MEMORY_INTER) %{
5896 base($reg);
5897 index($lreg);
5898 scale($scale);
5899 disp(0x0);
5900 %}
5901 %}
5902
5903 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
5904 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
5905 %{
5906 predicate(CompressedOops::shift() == 0);
5907 constraint(ALLOC_IN_RC(ptr_reg));
5908 match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
5909
5910 op_cost(10);
5911 format %{"[$reg + $off + $lreg << $scale]" %}
5912 interface(MEMORY_INTER) %{
5913 base($reg);
5914 index($lreg);
5915 scale($scale);
5916 disp($off);
5917 %}
5918 %}
5919
5920 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
5921 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
5922 %{
5923 constraint(ALLOC_IN_RC(ptr_reg));
5924 predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
5925 match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
5926
5927 op_cost(10);
5928 format %{"[$reg + $off + $idx]" %}
5929 interface(MEMORY_INTER) %{
5930 base($reg);
5931 index($idx);
5932 scale(0x0);
5933 disp($off);
5934 %}
5935 %}
5936
5937 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
5938 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
5939 %{
5940 constraint(ALLOC_IN_RC(ptr_reg));
5941 predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
5942 match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
5943
5944 op_cost(10);
5945 format %{"[$reg + $off + $idx << $scale]" %}
5946 interface(MEMORY_INTER) %{
5947 base($reg);
5948 index($idx);
5949 scale($scale);
5950 disp($off);
5951 %}
5952 %}
5953
5954 //----------Special Memory Operands--------------------------------------------
5955 // Stack Slot Operand - This operand is used for loading and storing temporary
5956 // values on the stack where a match requires a value to
5957 // flow through memory.
5958 operand stackSlotP(sRegP reg)
5959 %{
5960 constraint(ALLOC_IN_RC(stack_slots));
5961 // No match rule because this operand is only generated in matching
5962
5963 format %{ "[$reg]" %}
5964 interface(MEMORY_INTER) %{
5965 base(0x4); // RSP
5966 index(0x4); // No Index
5967 scale(0x0); // No Scale
5968 disp($reg); // Stack Offset
5969 %}
5970 %}
5971
5972 operand stackSlotI(sRegI reg)
5973 %{
5974 constraint(ALLOC_IN_RC(stack_slots));
5975 // No match rule because this operand is only generated in matching
5976
5977 format %{ "[$reg]" %}
5978 interface(MEMORY_INTER) %{
5979 base(0x4); // RSP
5980 index(0x4); // No Index
5981 scale(0x0); // No Scale
5982 disp($reg); // Stack Offset
5983 %}
5984 %}
5985
5986 operand stackSlotF(sRegF reg)
5987 %{
5988 constraint(ALLOC_IN_RC(stack_slots));
5989 // No match rule because this operand is only generated in matching
5990
5991 format %{ "[$reg]" %}
5992 interface(MEMORY_INTER) %{
5993 base(0x4); // RSP
5994 index(0x4); // No Index
5995 scale(0x0); // No Scale
5996 disp($reg); // Stack Offset
5997 %}
5998 %}
5999
6000 operand stackSlotD(sRegD reg)
6001 %{
6002 constraint(ALLOC_IN_RC(stack_slots));
6003 // No match rule because this operand is only generated in matching
6004
6005 format %{ "[$reg]" %}
6006 interface(MEMORY_INTER) %{
6007 base(0x4); // RSP
6008 index(0x4); // No Index
6009 scale(0x0); // No Scale
6010 disp($reg); // Stack Offset
6011 %}
6012 %}
6013 operand stackSlotL(sRegL reg)
6014 %{
6015 constraint(ALLOC_IN_RC(stack_slots));
6016 // No match rule because this operand is only generated in matching
6017
6018 format %{ "[$reg]" %}
6019 interface(MEMORY_INTER) %{
6020 base(0x4); // RSP
6021 index(0x4); // No Index
6022 scale(0x0); // No Scale
6023 disp($reg); // Stack Offset
6024 %}
6025 %}
6026
6027 //----------Conditional Branch Operands----------------------------------------
6028 // Comparison Op - This is the operation of the comparison, and is limited to
6029 // the following set of codes:
6030 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6031 //
6032 // Other attributes of the comparison, such as unsignedness, are specified
6033 // by the comparison instruction that sets a condition code flags register.
6034 // That result is represented by a flags operand whose subtype is appropriate
6035 // to the unsignedness (etc.) of the comparison.
6036 //
6037 // Later, the instruction which matches both the Comparison Op (a Bool) and
6038 // the flags (produced by the Cmp) specifies the coding of the comparison op
6039 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6040
6041 // Comparison Code
6042 operand cmpOp()
6043 %{
6044 match(Bool);
6045
6046 format %{ "" %}
6047 interface(COND_INTER) %{
6048 equal(0x4, "e");
6049 not_equal(0x5, "ne");
6050 less(0xc, "l");
6051 greater_equal(0xd, "ge");
6052 less_equal(0xe, "le");
6053 greater(0xf, "g");
6054 overflow(0x0, "o");
6055 no_overflow(0x1, "no");
6056 %}
6057 %}
6058
6059 // Comparison Code, unsigned compare. Used by FP also, with
6060 // C2 (unordered) turned into GT or LT already. The other bits
6061 // C0 and C3 are turned into Carry & Zero flags.
6062 operand cmpOpU()
6063 %{
6064 match(Bool);
6065
6066 format %{ "" %}
6067 interface(COND_INTER) %{
6068 equal(0x4, "e");
6069 not_equal(0x5, "ne");
6070 less(0x2, "b");
6071 greater_equal(0x3, "ae");
6072 less_equal(0x6, "be");
6073 greater(0x7, "a");
6074 overflow(0x0, "o");
6075 no_overflow(0x1, "no");
6076 %}
6077 %}
6078
6079
6080 // Floating comparisons that don't require any fixup for the unordered case,
6081 // If both inputs of the comparison are the same, ZF is always set so we
6082 // don't need to use cmpOpUCF2 for eq/ne
6083 operand cmpOpUCF() %{
6084 match(Bool);
6085 predicate((!UseAPX || !VM_Version::supports_avx10_2()) &&
6086 (n->as_Bool()->_test._test == BoolTest::lt ||
6087 n->as_Bool()->_test._test == BoolTest::ge ||
6088 n->as_Bool()->_test._test == BoolTest::le ||
6089 n->as_Bool()->_test._test == BoolTest::gt ||
6090 n->in(1)->in(1) == n->in(1)->in(2)));
6091 format %{ "" %}
6092 interface(COND_INTER) %{
6093 equal(0xb, "np");
6094 not_equal(0xa, "p");
6095 less(0x2, "b");
6096 greater_equal(0x3, "ae");
6097 less_equal(0x6, "be");
6098 greater(0x7, "a");
6099 overflow(0x0, "o");
6100 no_overflow(0x1, "no");
6101 %}
6102 %}
6103
6104
6105 // Floating comparisons that can be fixed up with extra conditional jumps
6106 operand cmpOpUCF2() %{
6107 match(Bool);
6108 predicate((!UseAPX || !VM_Version::supports_avx10_2()) &&
6109 (n->as_Bool()->_test._test == BoolTest::ne ||
6110 n->as_Bool()->_test._test == BoolTest::eq) &&
6111 n->in(1)->in(1) != n->in(1)->in(2));
6112 format %{ "" %}
6113 interface(COND_INTER) %{
6114 equal(0x4, "e");
6115 not_equal(0x5, "ne");
6116 less(0x2, "b");
6117 greater_equal(0x3, "ae");
6118 less_equal(0x6, "be");
6119 greater(0x7, "a");
6120 overflow(0x0, "o");
6121 no_overflow(0x1, "no");
6122 %}
6123 %}
6124
6125
6126 // Floating point comparisons that set condition flags to test more directly,
6127 // Unsigned tests are used for G (>) and GE (>=) conditions while signed tests
6128 // are used for L (<) and LE (<=) conditions. It's important to convert these
6129 // latter conditions to ones that use unsigned tests before passing into an
6130 // instruction because the preceding comparison might be based on a three way
6131 // comparison (CmpF3 or CmpD3) that also assigns unordered outcomes to -1.
6132 operand cmpOpUCFE()
6133 %{
6134 match(Bool);
6135 predicate((UseAPX && VM_Version::supports_avx10_2()) &&
6136 (n->as_Bool()->_test._test == BoolTest::ne ||
6137 n->as_Bool()->_test._test == BoolTest::eq ||
6138 n->as_Bool()->_test._test == BoolTest::lt ||
6139 n->as_Bool()->_test._test == BoolTest::ge ||
6140 n->as_Bool()->_test._test == BoolTest::le ||
6141 n->as_Bool()->_test._test == BoolTest::gt));
6142
6143 format %{ "" %}
6144 interface(COND_INTER) %{
6145 equal(0x4, "e");
6146 not_equal(0x5, "ne");
6147 less(0x2, "b");
6148 greater_equal(0x3, "ae");
6149 less_equal(0x6, "be");
6150 greater(0x7, "a");
6151 overflow(0x0, "o");
6152 no_overflow(0x1, "no");
6153 %}
6154 %}
6155
6156 // Operands for bound floating pointer register arguments
6157 operand rxmm0() %{
6158 constraint(ALLOC_IN_RC(xmm0_reg));
6159 match(VecX);
6160 format%{%}
6161 interface(REG_INTER);
6162 %}
6163
6164 // Vectors
6165
6166 // Dummy generic vector class. Should be used for all vector operands.
6167 // Replaced with vec[SDXYZ] during post-selection pass.
6168 operand vec() %{
6169 constraint(ALLOC_IN_RC(dynamic));
6170 match(VecX);
6171 match(VecY);
6172 match(VecZ);
6173 match(VecS);
6174 match(VecD);
6175
6176 format %{ %}
6177 interface(REG_INTER);
6178 %}
6179
6180 // Dummy generic legacy vector class. Should be used for all legacy vector operands.
6181 // Replaced with legVec[SDXYZ] during post-selection cleanup.
6182 // Note: legacy register class is used to avoid extra (unneeded in 32-bit VM)
6183 // runtime code generation via reg_class_dynamic.
6184 operand legVec() %{
6185 constraint(ALLOC_IN_RC(dynamic));
6186 match(VecX);
6187 match(VecY);
6188 match(VecZ);
6189 match(VecS);
6190 match(VecD);
6191
6192 format %{ %}
6193 interface(REG_INTER);
6194 %}
6195
6196 // Replaces vec during post-selection cleanup. See above.
6197 operand vecS() %{
6198 constraint(ALLOC_IN_RC(vectors_reg_vlbwdq));
6199 match(VecS);
6200
6201 format %{ %}
6202 interface(REG_INTER);
6203 %}
6204
6205 // Replaces legVec during post-selection cleanup. See above.
6206 operand legVecS() %{
6207 constraint(ALLOC_IN_RC(vectors_reg_legacy));
6208 match(VecS);
6209
6210 format %{ %}
6211 interface(REG_INTER);
6212 %}
6213
6214 // Replaces vec during post-selection cleanup. See above.
6215 operand vecD() %{
6216 constraint(ALLOC_IN_RC(vectord_reg_vlbwdq));
6217 match(VecD);
6218
6219 format %{ %}
6220 interface(REG_INTER);
6221 %}
6222
6223 // Replaces legVec during post-selection cleanup. See above.
6224 operand legVecD() %{
6225 constraint(ALLOC_IN_RC(vectord_reg_legacy));
6226 match(VecD);
6227
6228 format %{ %}
6229 interface(REG_INTER);
6230 %}
6231
6232 // Replaces vec during post-selection cleanup. See above.
6233 operand vecX() %{
6234 constraint(ALLOC_IN_RC(vectorx_reg_vlbwdq));
6235 match(VecX);
6236
6237 format %{ %}
6238 interface(REG_INTER);
6239 %}
6240
6241 // Replaces legVec during post-selection cleanup. See above.
6242 operand legVecX() %{
6243 constraint(ALLOC_IN_RC(vectorx_reg_legacy));
6244 match(VecX);
6245
6246 format %{ %}
6247 interface(REG_INTER);
6248 %}
6249
6250 // Replaces vec during post-selection cleanup. See above.
6251 operand vecY() %{
6252 constraint(ALLOC_IN_RC(vectory_reg_vlbwdq));
6253 match(VecY);
6254
6255 format %{ %}
6256 interface(REG_INTER);
6257 %}
6258
6259 // Replaces legVec during post-selection cleanup. See above.
6260 operand legVecY() %{
6261 constraint(ALLOC_IN_RC(vectory_reg_legacy));
6262 match(VecY);
6263
6264 format %{ %}
6265 interface(REG_INTER);
6266 %}
6267
6268 // Replaces vec during post-selection cleanup. See above.
6269 operand vecZ() %{
6270 constraint(ALLOC_IN_RC(vectorz_reg));
6271 match(VecZ);
6272
6273 format %{ %}
6274 interface(REG_INTER);
6275 %}
6276
6277 // Replaces legVec during post-selection cleanup. See above.
6278 operand legVecZ() %{
6279 constraint(ALLOC_IN_RC(vectorz_reg_legacy));
6280 match(VecZ);
6281
6282 format %{ %}
6283 interface(REG_INTER);
6284 %}
6285
6286 //----------OPERAND CLASSES----------------------------------------------------
6287 // Operand Classes are groups of operands that are used as to simplify
6288 // instruction definitions by not requiring the AD writer to specify separate
6289 // instructions for every form of operand when the instruction accepts
6290 // multiple operand types with the same basic encoding and format. The classic
6291 // case of this is memory operands.
6292
6293 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
6294 indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
6295 indCompressedOopOffset,
6296 indirectNarrow, indOffset8Narrow, indOffset32Narrow,
6297 indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
6298 indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
6299
6300 //----------PIPELINE-----------------------------------------------------------
6301 // Rules which define the behavior of the target architectures pipeline.
6302 pipeline %{
6303
6304 //----------ATTRIBUTES---------------------------------------------------------
6305 attributes %{
6306 variable_size_instructions; // Fixed size instructions
6307 max_instructions_per_bundle = 3; // Up to 3 instructions per bundle
6308 instruction_unit_size = 1; // An instruction is 1 bytes long
6309 instruction_fetch_unit_size = 16; // The processor fetches one line
6310 instruction_fetch_units = 1; // of 16 bytes
6311 %}
6312
6313 //----------RESOURCES----------------------------------------------------------
6314 // Resources are the functional units available to the machine
6315
6316 // Generic P2/P3 pipeline
6317 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
6318 // 3 instructions decoded per cycle.
6319 // 2 load/store ops per cycle, 1 branch, 1 FPU,
6320 // 3 ALU op, only ALU0 handles mul instructions.
6321 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
6322 MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
6323 BR, FPU,
6324 ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
6325
6326 //----------PIPELINE DESCRIPTION-----------------------------------------------
6327 // Pipeline Description specifies the stages in the machine's pipeline
6328
6329 // Generic P2/P3 pipeline
6330 pipe_desc(S0, S1, S2, S3, S4, S5);
6331
6332 //----------PIPELINE CLASSES---------------------------------------------------
6333 // Pipeline Classes describe the stages in which input and output are
6334 // referenced by the hardware pipeline.
6335
6336 // Naming convention: ialu or fpu
6337 // Then: _reg
6338 // Then: _reg if there is a 2nd register
6339 // Then: _long if it's a pair of instructions implementing a long
6340 // Then: _fat if it requires the big decoder
6341 // Or: _mem if it requires the big decoder and a memory unit.
6342
6343 // Integer ALU reg operation
6344 pipe_class ialu_reg(rRegI dst)
6345 %{
6346 single_instruction;
6347 dst : S4(write);
6348 dst : S3(read);
6349 DECODE : S0; // any decoder
6350 ALU : S3; // any alu
6351 %}
6352
6353 // Long ALU reg operation
6354 pipe_class ialu_reg_long(rRegL dst)
6355 %{
6356 instruction_count(2);
6357 dst : S4(write);
6358 dst : S3(read);
6359 DECODE : S0(2); // any 2 decoders
6360 ALU : S3(2); // both alus
6361 %}
6362
6363 // Integer ALU reg operation using big decoder
6364 pipe_class ialu_reg_fat(rRegI dst)
6365 %{
6366 single_instruction;
6367 dst : S4(write);
6368 dst : S3(read);
6369 D0 : S0; // big decoder only
6370 ALU : S3; // any alu
6371 %}
6372
6373 // Integer ALU reg-reg operation
6374 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
6375 %{
6376 single_instruction;
6377 dst : S4(write);
6378 src : S3(read);
6379 DECODE : S0; // any decoder
6380 ALU : S3; // any alu
6381 %}
6382
6383 // Integer ALU reg-reg operation
6384 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
6385 %{
6386 single_instruction;
6387 dst : S4(write);
6388 src : S3(read);
6389 D0 : S0; // big decoder only
6390 ALU : S3; // any alu
6391 %}
6392
6393 // Integer ALU reg-mem operation
6394 pipe_class ialu_reg_mem(rRegI dst, memory mem)
6395 %{
6396 single_instruction;
6397 dst : S5(write);
6398 mem : S3(read);
6399 D0 : S0; // big decoder only
6400 ALU : S4; // any alu
6401 MEM : S3; // any mem
6402 %}
6403
6404 // Integer mem operation (prefetch)
6405 pipe_class ialu_mem(memory mem)
6406 %{
6407 single_instruction;
6408 mem : S3(read);
6409 D0 : S0; // big decoder only
6410 MEM : S3; // any mem
6411 %}
6412
6413 // Integer Store to Memory
6414 pipe_class ialu_mem_reg(memory mem, rRegI src)
6415 %{
6416 single_instruction;
6417 mem : S3(read);
6418 src : S5(read);
6419 D0 : S0; // big decoder only
6420 ALU : S4; // any alu
6421 MEM : S3;
6422 %}
6423
6424 // // Long Store to Memory
6425 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
6426 // %{
6427 // instruction_count(2);
6428 // mem : S3(read);
6429 // src : S5(read);
6430 // D0 : S0(2); // big decoder only; twice
6431 // ALU : S4(2); // any 2 alus
6432 // MEM : S3(2); // Both mems
6433 // %}
6434
6435 // Integer Store to Memory
6436 pipe_class ialu_mem_imm(memory mem)
6437 %{
6438 single_instruction;
6439 mem : S3(read);
6440 D0 : S0; // big decoder only
6441 ALU : S4; // any alu
6442 MEM : S3;
6443 %}
6444
6445 // Integer ALU0 reg-reg operation
6446 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
6447 %{
6448 single_instruction;
6449 dst : S4(write);
6450 src : S3(read);
6451 D0 : S0; // Big decoder only
6452 ALU0 : S3; // only alu0
6453 %}
6454
6455 // Integer ALU0 reg-mem operation
6456 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
6457 %{
6458 single_instruction;
6459 dst : S5(write);
6460 mem : S3(read);
6461 D0 : S0; // big decoder only
6462 ALU0 : S4; // ALU0 only
6463 MEM : S3; // any mem
6464 %}
6465
6466 // Integer ALU reg-reg operation
6467 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
6468 %{
6469 single_instruction;
6470 cr : S4(write);
6471 src1 : S3(read);
6472 src2 : S3(read);
6473 DECODE : S0; // any decoder
6474 ALU : S3; // any alu
6475 %}
6476
6477 // Integer ALU reg-imm operation
6478 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
6479 %{
6480 single_instruction;
6481 cr : S4(write);
6482 src1 : S3(read);
6483 DECODE : S0; // any decoder
6484 ALU : S3; // any alu
6485 %}
6486
6487 // Integer ALU reg-mem operation
6488 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
6489 %{
6490 single_instruction;
6491 cr : S4(write);
6492 src1 : S3(read);
6493 src2 : S3(read);
6494 D0 : S0; // big decoder only
6495 ALU : S4; // any alu
6496 MEM : S3;
6497 %}
6498
6499 // Conditional move reg-reg
6500 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
6501 %{
6502 instruction_count(4);
6503 y : S4(read);
6504 q : S3(read);
6505 p : S3(read);
6506 DECODE : S0(4); // any decoder
6507 %}
6508
6509 // Conditional move reg-reg
6510 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
6511 %{
6512 single_instruction;
6513 dst : S4(write);
6514 src : S3(read);
6515 cr : S3(read);
6516 DECODE : S0; // any decoder
6517 %}
6518
6519 // Conditional move reg-mem
6520 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
6521 %{
6522 single_instruction;
6523 dst : S4(write);
6524 src : S3(read);
6525 cr : S3(read);
6526 DECODE : S0; // any decoder
6527 MEM : S3;
6528 %}
6529
6530 // Conditional move reg-reg long
6531 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
6532 %{
6533 single_instruction;
6534 dst : S4(write);
6535 src : S3(read);
6536 cr : S3(read);
6537 DECODE : S0(2); // any 2 decoders
6538 %}
6539
6540 // Float reg-reg operation
6541 pipe_class fpu_reg(regD dst)
6542 %{
6543 instruction_count(2);
6544 dst : S3(read);
6545 DECODE : S0(2); // any 2 decoders
6546 FPU : S3;
6547 %}
6548
6549 // Float reg-reg operation
6550 pipe_class fpu_reg_reg(regD dst, regD src)
6551 %{
6552 instruction_count(2);
6553 dst : S4(write);
6554 src : S3(read);
6555 DECODE : S0(2); // any 2 decoders
6556 FPU : S3;
6557 %}
6558
6559 // Float reg-reg operation
6560 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
6561 %{
6562 instruction_count(3);
6563 dst : S4(write);
6564 src1 : S3(read);
6565 src2 : S3(read);
6566 DECODE : S0(3); // any 3 decoders
6567 FPU : S3(2);
6568 %}
6569
6570 // Float reg-reg operation
6571 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
6572 %{
6573 instruction_count(4);
6574 dst : S4(write);
6575 src1 : S3(read);
6576 src2 : S3(read);
6577 src3 : S3(read);
6578 DECODE : S0(4); // any 3 decoders
6579 FPU : S3(2);
6580 %}
6581
6582 // Float reg-reg operation
6583 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
6584 %{
6585 instruction_count(4);
6586 dst : S4(write);
6587 src1 : S3(read);
6588 src2 : S3(read);
6589 src3 : S3(read);
6590 DECODE : S1(3); // any 3 decoders
6591 D0 : S0; // Big decoder only
6592 FPU : S3(2);
6593 MEM : S3;
6594 %}
6595
6596 // Float reg-mem operation
6597 pipe_class fpu_reg_mem(regD dst, memory mem)
6598 %{
6599 instruction_count(2);
6600 dst : S5(write);
6601 mem : S3(read);
6602 D0 : S0; // big decoder only
6603 DECODE : S1; // any decoder for FPU POP
6604 FPU : S4;
6605 MEM : S3; // any mem
6606 %}
6607
6608 // Float reg-mem operation
6609 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
6610 %{
6611 instruction_count(3);
6612 dst : S5(write);
6613 src1 : S3(read);
6614 mem : S3(read);
6615 D0 : S0; // big decoder only
6616 DECODE : S1(2); // any decoder for FPU POP
6617 FPU : S4;
6618 MEM : S3; // any mem
6619 %}
6620
6621 // Float mem-reg operation
6622 pipe_class fpu_mem_reg(memory mem, regD src)
6623 %{
6624 instruction_count(2);
6625 src : S5(read);
6626 mem : S3(read);
6627 DECODE : S0; // any decoder for FPU PUSH
6628 D0 : S1; // big decoder only
6629 FPU : S4;
6630 MEM : S3; // any mem
6631 %}
6632
6633 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
6634 %{
6635 instruction_count(3);
6636 src1 : S3(read);
6637 src2 : S3(read);
6638 mem : S3(read);
6639 DECODE : S0(2); // any decoder for FPU PUSH
6640 D0 : S1; // big decoder only
6641 FPU : S4;
6642 MEM : S3; // any mem
6643 %}
6644
6645 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
6646 %{
6647 instruction_count(3);
6648 src1 : S3(read);
6649 src2 : S3(read);
6650 mem : S4(read);
6651 DECODE : S0; // any decoder for FPU PUSH
6652 D0 : S0(2); // big decoder only
6653 FPU : S4;
6654 MEM : S3(2); // any mem
6655 %}
6656
6657 pipe_class fpu_mem_mem(memory dst, memory src1)
6658 %{
6659 instruction_count(2);
6660 src1 : S3(read);
6661 dst : S4(read);
6662 D0 : S0(2); // big decoder only
6663 MEM : S3(2); // any mem
6664 %}
6665
6666 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
6667 %{
6668 instruction_count(3);
6669 src1 : S3(read);
6670 src2 : S3(read);
6671 dst : S4(read);
6672 D0 : S0(3); // big decoder only
6673 FPU : S4;
6674 MEM : S3(3); // any mem
6675 %}
6676
6677 pipe_class fpu_mem_reg_con(memory mem, regD src1)
6678 %{
6679 instruction_count(3);
6680 src1 : S4(read);
6681 mem : S4(read);
6682 DECODE : S0; // any decoder for FPU PUSH
6683 D0 : S0(2); // big decoder only
6684 FPU : S4;
6685 MEM : S3(2); // any mem
6686 %}
6687
6688 // Float load constant
6689 pipe_class fpu_reg_con(regD dst)
6690 %{
6691 instruction_count(2);
6692 dst : S5(write);
6693 D0 : S0; // big decoder only for the load
6694 DECODE : S1; // any decoder for FPU POP
6695 FPU : S4;
6696 MEM : S3; // any mem
6697 %}
6698
6699 // Float load constant
6700 pipe_class fpu_reg_reg_con(regD dst, regD src)
6701 %{
6702 instruction_count(3);
6703 dst : S5(write);
6704 src : S3(read);
6705 D0 : S0; // big decoder only for the load
6706 DECODE : S1(2); // any decoder for FPU POP
6707 FPU : S4;
6708 MEM : S3; // any mem
6709 %}
6710
6711 // UnConditional branch
6712 pipe_class pipe_jmp(label labl)
6713 %{
6714 single_instruction;
6715 BR : S3;
6716 %}
6717
6718 // Conditional branch
6719 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
6720 %{
6721 single_instruction;
6722 cr : S1(read);
6723 BR : S3;
6724 %}
6725
6726 // Allocation idiom
6727 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
6728 %{
6729 instruction_count(1); force_serialization;
6730 fixed_latency(6);
6731 heap_ptr : S3(read);
6732 DECODE : S0(3);
6733 D0 : S2;
6734 MEM : S3;
6735 ALU : S3(2);
6736 dst : S5(write);
6737 BR : S5;
6738 %}
6739
6740 // Generic big/slow expanded idiom
6741 pipe_class pipe_slow()
6742 %{
6743 instruction_count(10); multiple_bundles; force_serialization;
6744 fixed_latency(100);
6745 D0 : S0(2);
6746 MEM : S3(2);
6747 %}
6748
6749 // The real do-nothing guy
6750 pipe_class empty()
6751 %{
6752 instruction_count(0);
6753 %}
6754
6755 // Define the class for the Nop node
6756 define
6757 %{
6758 MachNop = empty;
6759 %}
6760
6761 %}
6762
6763 //----------INSTRUCTIONS-------------------------------------------------------
6764 //
6765 // match -- States which machine-independent subtree may be replaced
6766 // by this instruction.
6767 // ins_cost -- The estimated cost of this instruction is used by instruction
6768 // selection to identify a minimum cost tree of machine
6769 // instructions that matches a tree of machine-independent
6770 // instructions.
6771 // format -- A string providing the disassembly for this instruction.
6772 // The value of an instruction's operand may be inserted
6773 // by referring to it with a '$' prefix.
6774 // opcode -- Three instruction opcodes may be provided. These are referred
6775 // to within an encode class as $primary, $secondary, and $tertiary
6776 // rrspectively. The primary opcode is commonly used to
6777 // indicate the type of machine instruction, while secondary
6778 // and tertiary are often used for prefix options or addressing
6779 // modes.
6780 // ins_encode -- A list of encode classes with parameters. The encode class
6781 // name must have been defined in an 'enc_class' specification
6782 // in the encode section of the architecture description.
6783
6784 // ============================================================================
6785
6786 instruct ShouldNotReachHere() %{
6787 match(Halt);
6788 format %{ "stop\t# ShouldNotReachHere" %}
6789 ins_encode %{
6790 if (is_reachable()) {
6791 const char* str = __ code_string(_halt_reason);
6792 __ stop(str);
6793 }
6794 %}
6795 ins_pipe(pipe_slow);
6796 %}
6797
6798 // ============================================================================
6799
6800 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
6801 // Load Float
6802 instruct MoveF2VL(vlRegF dst, regF src) %{
6803 match(Set dst src);
6804 format %{ "movss $dst,$src\t! load float (4 bytes)" %}
6805 ins_encode %{
6806 ShouldNotReachHere();
6807 %}
6808 ins_pipe( fpu_reg_reg );
6809 %}
6810
6811 // Load Float
6812 instruct MoveF2LEG(legRegF dst, regF src) %{
6813 match(Set dst src);
6814 format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
6815 ins_encode %{
6816 ShouldNotReachHere();
6817 %}
6818 ins_pipe( fpu_reg_reg );
6819 %}
6820
6821 // Load Float
6822 instruct MoveVL2F(regF dst, vlRegF src) %{
6823 match(Set dst src);
6824 format %{ "movss $dst,$src\t! load float (4 bytes)" %}
6825 ins_encode %{
6826 ShouldNotReachHere();
6827 %}
6828 ins_pipe( fpu_reg_reg );
6829 %}
6830
6831 // Load Float
6832 instruct MoveLEG2F(regF dst, legRegF src) %{
6833 match(Set dst src);
6834 format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
6835 ins_encode %{
6836 ShouldNotReachHere();
6837 %}
6838 ins_pipe( fpu_reg_reg );
6839 %}
6840
6841 // Load Double
6842 instruct MoveD2VL(vlRegD dst, regD src) %{
6843 match(Set dst src);
6844 format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
6845 ins_encode %{
6846 ShouldNotReachHere();
6847 %}
6848 ins_pipe( fpu_reg_reg );
6849 %}
6850
6851 // Load Double
6852 instruct MoveD2LEG(legRegD dst, regD src) %{
6853 match(Set dst src);
6854 format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
6855 ins_encode %{
6856 ShouldNotReachHere();
6857 %}
6858 ins_pipe( fpu_reg_reg );
6859 %}
6860
6861 // Load Double
6862 instruct MoveVL2D(regD dst, vlRegD src) %{
6863 match(Set dst src);
6864 format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
6865 ins_encode %{
6866 ShouldNotReachHere();
6867 %}
6868 ins_pipe( fpu_reg_reg );
6869 %}
6870
6871 // Load Double
6872 instruct MoveLEG2D(regD dst, legRegD src) %{
6873 match(Set dst src);
6874 format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
6875 ins_encode %{
6876 ShouldNotReachHere();
6877 %}
6878 ins_pipe( fpu_reg_reg );
6879 %}
6880
6881 //----------Load/Store/Move Instructions---------------------------------------
6882 //----------Load Instructions--------------------------------------------------
6883
6884 // Load Byte (8 bit signed)
6885 instruct loadB(rRegI dst, memory mem)
6886 %{
6887 match(Set dst (LoadB mem));
6888
6889 ins_cost(125);
6890 format %{ "movsbl $dst, $mem\t# byte" %}
6891
6892 ins_encode %{
6893 __ movsbl($dst$$Register, $mem$$Address);
6894 %}
6895
6896 ins_pipe(ialu_reg_mem);
6897 %}
6898
6899 // Load Byte (8 bit signed) into Long Register
6900 instruct loadB2L(rRegL dst, memory mem)
6901 %{
6902 match(Set dst (ConvI2L (LoadB mem)));
6903
6904 ins_cost(125);
6905 format %{ "movsbq $dst, $mem\t# byte -> long" %}
6906
6907 ins_encode %{
6908 __ movsbq($dst$$Register, $mem$$Address);
6909 %}
6910
6911 ins_pipe(ialu_reg_mem);
6912 %}
6913
6914 // Load Unsigned Byte (8 bit UNsigned)
6915 instruct loadUB(rRegI dst, memory mem)
6916 %{
6917 match(Set dst (LoadUB mem));
6918
6919 ins_cost(125);
6920 format %{ "movzbl $dst, $mem\t# ubyte" %}
6921
6922 ins_encode %{
6923 __ movzbl($dst$$Register, $mem$$Address);
6924 %}
6925
6926 ins_pipe(ialu_reg_mem);
6927 %}
6928
6929 // Load Unsigned Byte (8 bit UNsigned) into Long Register
6930 instruct loadUB2L(rRegL dst, memory mem)
6931 %{
6932 match(Set dst (ConvI2L (LoadUB mem)));
6933
6934 ins_cost(125);
6935 format %{ "movzbq $dst, $mem\t# ubyte -> long" %}
6936
6937 ins_encode %{
6938 __ movzbq($dst$$Register, $mem$$Address);
6939 %}
6940
6941 ins_pipe(ialu_reg_mem);
6942 %}
6943
6944 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
6945 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
6946 match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
6947 effect(KILL cr);
6948
6949 format %{ "movzbq $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
6950 "andl $dst, right_n_bits($mask, 8)" %}
6951 ins_encode %{
6952 Register Rdst = $dst$$Register;
6953 __ movzbq(Rdst, $mem$$Address);
6954 __ andl(Rdst, $mask$$constant & right_n_bits(8));
6955 %}
6956 ins_pipe(ialu_reg_mem);
6957 %}
6958
6959 // Load Short (16 bit signed)
6960 instruct loadS(rRegI dst, memory mem)
6961 %{
6962 match(Set dst (LoadS mem));
6963
6964 ins_cost(125);
6965 format %{ "movswl $dst, $mem\t# short" %}
6966
6967 ins_encode %{
6968 __ movswl($dst$$Register, $mem$$Address);
6969 %}
6970
6971 ins_pipe(ialu_reg_mem);
6972 %}
6973
6974 // Load Short (16 bit signed) to Byte (8 bit signed)
6975 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
6976 match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
6977
6978 ins_cost(125);
6979 format %{ "movsbl $dst, $mem\t# short -> byte" %}
6980 ins_encode %{
6981 __ movsbl($dst$$Register, $mem$$Address);
6982 %}
6983 ins_pipe(ialu_reg_mem);
6984 %}
6985
6986 // Load Short (16 bit signed) into Long Register
6987 instruct loadS2L(rRegL dst, memory mem)
6988 %{
6989 match(Set dst (ConvI2L (LoadS mem)));
6990
6991 ins_cost(125);
6992 format %{ "movswq $dst, $mem\t# short -> long" %}
6993
6994 ins_encode %{
6995 __ movswq($dst$$Register, $mem$$Address);
6996 %}
6997
6998 ins_pipe(ialu_reg_mem);
6999 %}
7000
7001 // Load Unsigned Short/Char (16 bit UNsigned)
7002 instruct loadUS(rRegI dst, memory mem)
7003 %{
7004 match(Set dst (LoadUS mem));
7005
7006 ins_cost(125);
7007 format %{ "movzwl $dst, $mem\t# ushort/char" %}
7008
7009 ins_encode %{
7010 __ movzwl($dst$$Register, $mem$$Address);
7011 %}
7012
7013 ins_pipe(ialu_reg_mem);
7014 %}
7015
7016 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
7017 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
7018 match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
7019
7020 ins_cost(125);
7021 format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
7022 ins_encode %{
7023 __ movsbl($dst$$Register, $mem$$Address);
7024 %}
7025 ins_pipe(ialu_reg_mem);
7026 %}
7027
7028 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
7029 instruct loadUS2L(rRegL dst, memory mem)
7030 %{
7031 match(Set dst (ConvI2L (LoadUS mem)));
7032
7033 ins_cost(125);
7034 format %{ "movzwq $dst, $mem\t# ushort/char -> long" %}
7035
7036 ins_encode %{
7037 __ movzwq($dst$$Register, $mem$$Address);
7038 %}
7039
7040 ins_pipe(ialu_reg_mem);
7041 %}
7042
7043 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
7044 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
7045 match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
7046
7047 format %{ "movzbq $dst, $mem\t# ushort/char & 0xFF -> long" %}
7048 ins_encode %{
7049 __ movzbq($dst$$Register, $mem$$Address);
7050 %}
7051 ins_pipe(ialu_reg_mem);
7052 %}
7053
7054 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
7055 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
7056 match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
7057 effect(KILL cr);
7058
7059 format %{ "movzwq $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
7060 "andl $dst, right_n_bits($mask, 16)" %}
7061 ins_encode %{
7062 Register Rdst = $dst$$Register;
7063 __ movzwq(Rdst, $mem$$Address);
7064 __ andl(Rdst, $mask$$constant & right_n_bits(16));
7065 %}
7066 ins_pipe(ialu_reg_mem);
7067 %}
7068
7069 // Load Integer
7070 instruct loadI(rRegI dst, memory mem)
7071 %{
7072 match(Set dst (LoadI mem));
7073
7074 ins_cost(125);
7075 format %{ "movl $dst, $mem\t# int" %}
7076
7077 ins_encode %{
7078 __ movl($dst$$Register, $mem$$Address);
7079 %}
7080
7081 ins_pipe(ialu_reg_mem);
7082 %}
7083
7084 // Load Integer (32 bit signed) to Byte (8 bit signed)
7085 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
7086 match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
7087
7088 ins_cost(125);
7089 format %{ "movsbl $dst, $mem\t# int -> byte" %}
7090 ins_encode %{
7091 __ movsbl($dst$$Register, $mem$$Address);
7092 %}
7093 ins_pipe(ialu_reg_mem);
7094 %}
7095
7096 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
7097 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
7098 match(Set dst (AndI (LoadI mem) mask));
7099
7100 ins_cost(125);
7101 format %{ "movzbl $dst, $mem\t# int -> ubyte" %}
7102 ins_encode %{
7103 __ movzbl($dst$$Register, $mem$$Address);
7104 %}
7105 ins_pipe(ialu_reg_mem);
7106 %}
7107
7108 // Load Integer (32 bit signed) to Short (16 bit signed)
7109 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
7110 match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
7111
7112 ins_cost(125);
7113 format %{ "movswl $dst, $mem\t# int -> short" %}
7114 ins_encode %{
7115 __ movswl($dst$$Register, $mem$$Address);
7116 %}
7117 ins_pipe(ialu_reg_mem);
7118 %}
7119
7120 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
7121 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
7122 match(Set dst (AndI (LoadI mem) mask));
7123
7124 ins_cost(125);
7125 format %{ "movzwl $dst, $mem\t# int -> ushort/char" %}
7126 ins_encode %{
7127 __ movzwl($dst$$Register, $mem$$Address);
7128 %}
7129 ins_pipe(ialu_reg_mem);
7130 %}
7131
7132 // Load Integer into Long Register
7133 instruct loadI2L(rRegL dst, memory mem)
7134 %{
7135 match(Set dst (ConvI2L (LoadI mem)));
7136
7137 ins_cost(125);
7138 format %{ "movslq $dst, $mem\t# int -> long" %}
7139
7140 ins_encode %{
7141 __ movslq($dst$$Register, $mem$$Address);
7142 %}
7143
7144 ins_pipe(ialu_reg_mem);
7145 %}
7146
7147 // Load Integer with mask 0xFF into Long Register
7148 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
7149 match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
7150
7151 format %{ "movzbq $dst, $mem\t# int & 0xFF -> long" %}
7152 ins_encode %{
7153 __ movzbq($dst$$Register, $mem$$Address);
7154 %}
7155 ins_pipe(ialu_reg_mem);
7156 %}
7157
7158 // Load Integer with mask 0xFFFF into Long Register
7159 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
7160 match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
7161
7162 format %{ "movzwq $dst, $mem\t# int & 0xFFFF -> long" %}
7163 ins_encode %{
7164 __ movzwq($dst$$Register, $mem$$Address);
7165 %}
7166 ins_pipe(ialu_reg_mem);
7167 %}
7168
7169 // Load Integer with a 31-bit mask into Long Register
7170 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
7171 match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
7172 effect(KILL cr);
7173
7174 format %{ "movl $dst, $mem\t# int & 31-bit mask -> long\n\t"
7175 "andl $dst, $mask" %}
7176 ins_encode %{
7177 Register Rdst = $dst$$Register;
7178 __ movl(Rdst, $mem$$Address);
7179 __ andl(Rdst, $mask$$constant);
7180 %}
7181 ins_pipe(ialu_reg_mem);
7182 %}
7183
7184 // Load Unsigned Integer into Long Register
7185 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
7186 %{
7187 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7188
7189 ins_cost(125);
7190 format %{ "movl $dst, $mem\t# uint -> long" %}
7191
7192 ins_encode %{
7193 __ movl($dst$$Register, $mem$$Address);
7194 %}
7195
7196 ins_pipe(ialu_reg_mem);
7197 %}
7198
7199 // Load Long
7200 instruct loadL(rRegL dst, memory mem)
7201 %{
7202 match(Set dst (LoadL mem));
7203
7204 ins_cost(125);
7205 format %{ "movq $dst, $mem\t# long" %}
7206
7207 ins_encode %{
7208 __ movq($dst$$Register, $mem$$Address);
7209 %}
7210
7211 ins_pipe(ialu_reg_mem); // XXX
7212 %}
7213
7214 // Load Range
7215 instruct loadRange(rRegI dst, memory mem)
7216 %{
7217 match(Set dst (LoadRange mem));
7218
7219 ins_cost(125); // XXX
7220 format %{ "movl $dst, $mem\t# range" %}
7221 ins_encode %{
7222 __ movl($dst$$Register, $mem$$Address);
7223 %}
7224 ins_pipe(ialu_reg_mem);
7225 %}
7226
7227 // Load Pointer
7228 instruct loadP(rRegP dst, memory mem)
7229 %{
7230 match(Set dst (LoadP mem));
7231 predicate(n->as_Load()->barrier_data() == 0);
7232
7233 ins_cost(125); // XXX
7234 format %{ "movq $dst, $mem\t# ptr" %}
7235 ins_encode %{
7236 __ movq($dst$$Register, $mem$$Address);
7237 %}
7238 ins_pipe(ialu_reg_mem); // XXX
7239 %}
7240
7241 // Load Compressed Pointer
7242 instruct loadN(rRegN dst, memory mem)
7243 %{
7244 predicate(n->as_Load()->barrier_data() == 0);
7245 match(Set dst (LoadN mem));
7246
7247 ins_cost(125); // XXX
7248 format %{ "movl $dst, $mem\t# compressed ptr" %}
7249 ins_encode %{
7250 __ movl($dst$$Register, $mem$$Address);
7251 %}
7252 ins_pipe(ialu_reg_mem); // XXX
7253 %}
7254
7255
7256 // Load Klass Pointer
7257 instruct loadKlass(rRegP dst, memory mem)
7258 %{
7259 match(Set dst (LoadKlass mem));
7260
7261 ins_cost(125); // XXX
7262 format %{ "movq $dst, $mem\t# class" %}
7263 ins_encode %{
7264 __ movq($dst$$Register, $mem$$Address);
7265 %}
7266 ins_pipe(ialu_reg_mem); // XXX
7267 %}
7268
7269 // Load narrow Klass Pointer
7270 instruct loadNKlass(rRegN dst, memory mem)
7271 %{
7272 predicate(!UseCompactObjectHeaders);
7273 match(Set dst (LoadNKlass mem));
7274
7275 ins_cost(125); // XXX
7276 format %{ "movl $dst, $mem\t# compressed klass ptr" %}
7277 ins_encode %{
7278 __ movl($dst$$Register, $mem$$Address);
7279 %}
7280 ins_pipe(ialu_reg_mem); // XXX
7281 %}
7282
7283 instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr)
7284 %{
7285 predicate(UseCompactObjectHeaders);
7286 match(Set dst (LoadNKlass mem));
7287 effect(KILL cr);
7288 ins_cost(125);
7289 format %{
7290 "movl $dst, $mem\t# compressed klass ptr, shifted\n\t"
7291 "shrl $dst, markWord::klass_shift_at_offset"
7292 %}
7293 ins_encode %{
7294 __ movl($dst$$Register, $mem$$Address);
7295 __ shrl($dst$$Register, markWord::klass_shift_at_offset);
7296 %}
7297 ins_pipe(ialu_reg_mem);
7298 %}
7299
7300 // Load Float
7301 instruct loadF(regF dst, memory mem)
7302 %{
7303 match(Set dst (LoadF mem));
7304
7305 ins_cost(145); // XXX
7306 format %{ "movss $dst, $mem\t# float" %}
7307 ins_encode %{
7308 __ movflt($dst$$XMMRegister, $mem$$Address);
7309 %}
7310 ins_pipe(pipe_slow); // XXX
7311 %}
7312
7313 // Load Double
7314 instruct loadD_partial(regD dst, memory mem)
7315 %{
7316 predicate(!UseXmmLoadAndClearUpper);
7317 match(Set dst (LoadD mem));
7318
7319 ins_cost(145); // XXX
7320 format %{ "movlpd $dst, $mem\t# double" %}
7321 ins_encode %{
7322 __ movdbl($dst$$XMMRegister, $mem$$Address);
7323 %}
7324 ins_pipe(pipe_slow); // XXX
7325 %}
7326
7327 instruct loadD(regD dst, memory mem)
7328 %{
7329 predicate(UseXmmLoadAndClearUpper);
7330 match(Set dst (LoadD mem));
7331
7332 ins_cost(145); // XXX
7333 format %{ "movsd $dst, $mem\t# double" %}
7334 ins_encode %{
7335 __ movdbl($dst$$XMMRegister, $mem$$Address);
7336 %}
7337 ins_pipe(pipe_slow); // XXX
7338 %}
7339
7340 instruct loadAOTRCAddress(rRegP dst, immAOTRuntimeConstantsAddress con)
7341 %{
7342 match(Set dst con);
7343
7344 format %{ "leaq $dst, $con\t# AOT Runtime Constants Address" %}
7345
7346 ins_encode %{
7347 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
7348 %}
7349
7350 ins_pipe(ialu_reg_fat);
7351 %}
7352
7353 // min = java.lang.Math.min(float a, float b)
7354 // max = java.lang.Math.max(float a, float b)
7355 instruct minmaxF_reg_avx10_2(regF dst, regF a, regF b)
7356 %{
7357 predicate(VM_Version::supports_avx10_2() && !VLoopReductions::is_reduction(n));
7358 match(Set dst (MaxF a b));
7359 match(Set dst (MinF a b));
7360
7361 format %{ "minmaxF $dst, $a, $b" %}
7362 ins_encode %{
7363 int opcode = this->ideal_Opcode();
7364 __ sminmax_fp_avx10_2(opcode, T_FLOAT, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister);
7365 %}
7366 ins_pipe( pipe_slow );
7367 %}
7368
7369 instruct minmaxF_reduction_reg_avx10_2(regF dst, regF a, regF b, rRegI rtmp, rFlagsReg cr)
7370 %{
7371 predicate(VM_Version::supports_avx10_2() && VLoopReductions::is_reduction(n));
7372 match(Set dst (MaxF a b));
7373 match(Set dst (MinF a b));
7374 effect(USE a, USE b, TEMP rtmp, KILL cr);
7375
7376 format %{ "minmaxF_reduction $dst, $a, $b \t! using $rtmp as TEMP" %}
7377 ins_encode %{
7378 int opcode = this->ideal_Opcode();
7379 bool min = (opcode == Op_MinF) ? true : false;
7380 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7381 min, fp_prec_flt /*pt*/);
7382 %}
7383 ins_pipe( pipe_slow );
7384 %}
7385
7386 // min = java.lang.Math.min(float a, float b)
7387 // max = java.lang.Math.max(float a, float b)
7388 instruct minmaxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp)
7389 %{
7390 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
7391 match(Set dst (MaxF a b));
7392 match(Set dst (MinF a b));
7393 effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
7394
7395 format %{ "minmaxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
7396 ins_encode %{
7397 int opcode = this->ideal_Opcode();
7398 int param_opcode = (opcode == Op_MinF) ? Op_MinV : Op_MaxV;
7399 __ vminmax_fp(param_opcode, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister,
7400 $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
7401 %}
7402 ins_pipe( pipe_slow );
7403 %}
7404
7405 instruct minmaxF_reduction_reg(legRegF dst, legRegF a, legRegF b, rRegI rtmp, rFlagsReg cr)
7406 %{
7407 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
7408 match(Set dst (MaxF a b));
7409 match(Set dst (MinF a b));
7410 effect(USE a, USE b, TEMP rtmp, KILL cr);
7411
7412 format %{ "minmaxF_reduction $dst, $a, $b \t!using $rtmp as TEMP" %}
7413 ins_encode %{
7414 int opcode = this->ideal_Opcode();
7415 bool min = (opcode == Op_MinF) ? true : false;
7416 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7417 min, fp_prec_flt /*pt*/);
7418 %}
7419 ins_pipe( pipe_slow );
7420 %}
7421
7422 // min = java.lang.Math.min(double a, double b)
7423 // max = java.lang.Math.max(double a, double b)
7424 instruct minmaxD_reg_avx10_2(regD dst, regD a, regD b)
7425 %{
7426 predicate(VM_Version::supports_avx10_2() && !VLoopReductions::is_reduction(n));
7427 match(Set dst (MaxD a b));
7428 match(Set dst (MinD a b));
7429
7430 format %{ "minmaxD $dst, $a, $b" %}
7431 ins_encode %{
7432 int opcode = this->ideal_Opcode();
7433 __ sminmax_fp_avx10_2(opcode, T_DOUBLE, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister);
7434 %}
7435 ins_pipe( pipe_slow );
7436 %}
7437
7438 instruct minmaxD_reduction_reg_avx10_2(regD dst, regD a, regD b, rRegI rtmp, rFlagsReg cr)
7439 %{
7440 predicate(VM_Version::supports_avx10_2() && VLoopReductions::is_reduction(n));
7441 match(Set dst (MaxD a b));
7442 match(Set dst (MinD a b));
7443 effect(USE a, USE b, TEMP rtmp, KILL cr);
7444
7445 format %{ "minmaxD_reduction $dst, $a, $b \t! using $rtmp as TEMP" %}
7446 ins_encode %{
7447 int opcode = this->ideal_Opcode();
7448 bool min = (opcode == Op_MinD) ? true : false;
7449 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7450 min, fp_prec_dbl /*pt*/);
7451 %}
7452 ins_pipe( pipe_slow );
7453 %}
7454
7455 // min = java.lang.Math.min(double a, double b)
7456 // max = java.lang.Math.max(double a, double b)
7457 instruct minmaxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp)
7458 %{
7459 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
7460 match(Set dst (MaxD a b));
7461 match(Set dst (MinD a b));
7462 effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
7463
7464 format %{ "minmaxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
7465 ins_encode %{
7466 int opcode = this->ideal_Opcode();
7467 int param_opcode = (opcode == Op_MinD) ? Op_MinV : Op_MaxV;
7468 __ vminmax_fp(param_opcode, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister,
7469 $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
7470 %}
7471 ins_pipe( pipe_slow );
7472 %}
7473
7474 instruct minmaxD_reduction_reg(legRegD dst, legRegD a, legRegD b, rRegL rtmp, rFlagsReg cr)
7475 %{
7476 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
7477 match(Set dst (MaxD a b));
7478 match(Set dst (MinD a b));
7479 effect(USE a, USE b, TEMP rtmp, KILL cr);
7480
7481 format %{ "minmaxD_reduction $dst, $a, $b \t! using $rtmp as TEMP" %}
7482 ins_encode %{
7483 int opcode = this->ideal_Opcode();
7484 bool min = (opcode == Op_MinD) ? true : false;
7485 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7486 min, fp_prec_dbl /*pt*/);
7487 %}
7488 ins_pipe( pipe_slow );
7489 %}
7490
7491 // Load Effective Address
7492 instruct leaP8(rRegP dst, indOffset8 mem)
7493 %{
7494 match(Set dst mem);
7495
7496 ins_cost(110); // XXX
7497 format %{ "leaq $dst, $mem\t# ptr 8" %}
7498 ins_encode %{
7499 __ leaq($dst$$Register, $mem$$Address);
7500 %}
7501 ins_pipe(ialu_reg_reg_fat);
7502 %}
7503
7504 instruct leaP32(rRegP dst, indOffset32 mem)
7505 %{
7506 match(Set dst mem);
7507
7508 ins_cost(110);
7509 format %{ "leaq $dst, $mem\t# ptr 32" %}
7510 ins_encode %{
7511 __ leaq($dst$$Register, $mem$$Address);
7512 %}
7513 ins_pipe(ialu_reg_reg_fat);
7514 %}
7515
7516 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
7517 %{
7518 match(Set dst mem);
7519
7520 ins_cost(110);
7521 format %{ "leaq $dst, $mem\t# ptr idxoff" %}
7522 ins_encode %{
7523 __ leaq($dst$$Register, $mem$$Address);
7524 %}
7525 ins_pipe(ialu_reg_reg_fat);
7526 %}
7527
7528 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
7529 %{
7530 match(Set dst mem);
7531
7532 ins_cost(110);
7533 format %{ "leaq $dst, $mem\t# ptr idxscale" %}
7534 ins_encode %{
7535 __ leaq($dst$$Register, $mem$$Address);
7536 %}
7537 ins_pipe(ialu_reg_reg_fat);
7538 %}
7539
7540 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
7541 %{
7542 match(Set dst mem);
7543
7544 ins_cost(110);
7545 format %{ "leaq $dst, $mem\t# ptr idxscale" %}
7546 ins_encode %{
7547 __ leaq($dst$$Register, $mem$$Address);
7548 %}
7549 ins_pipe(ialu_reg_reg_fat);
7550 %}
7551
7552 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
7553 %{
7554 match(Set dst mem);
7555
7556 ins_cost(110);
7557 format %{ "leaq $dst, $mem\t# ptr idxscaleoff" %}
7558 ins_encode %{
7559 __ leaq($dst$$Register, $mem$$Address);
7560 %}
7561 ins_pipe(ialu_reg_reg_fat);
7562 %}
7563
7564 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
7565 %{
7566 match(Set dst mem);
7567
7568 ins_cost(110);
7569 format %{ "leaq $dst, $mem\t# ptr posidxoff" %}
7570 ins_encode %{
7571 __ leaq($dst$$Register, $mem$$Address);
7572 %}
7573 ins_pipe(ialu_reg_reg_fat);
7574 %}
7575
7576 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
7577 %{
7578 match(Set dst mem);
7579
7580 ins_cost(110);
7581 format %{ "leaq $dst, $mem\t# ptr posidxscaleoff" %}
7582 ins_encode %{
7583 __ leaq($dst$$Register, $mem$$Address);
7584 %}
7585 ins_pipe(ialu_reg_reg_fat);
7586 %}
7587
7588 // Load Effective Address which uses Narrow (32-bits) oop
7589 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
7590 %{
7591 predicate(UseCompressedOops && (CompressedOops::shift() != 0));
7592 match(Set dst mem);
7593
7594 ins_cost(110);
7595 format %{ "leaq $dst, $mem\t# ptr compressedoopoff32" %}
7596 ins_encode %{
7597 __ leaq($dst$$Register, $mem$$Address);
7598 %}
7599 ins_pipe(ialu_reg_reg_fat);
7600 %}
7601
7602 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
7603 %{
7604 predicate(CompressedOops::shift() == 0);
7605 match(Set dst mem);
7606
7607 ins_cost(110); // XXX
7608 format %{ "leaq $dst, $mem\t# ptr off8narrow" %}
7609 ins_encode %{
7610 __ leaq($dst$$Register, $mem$$Address);
7611 %}
7612 ins_pipe(ialu_reg_reg_fat);
7613 %}
7614
7615 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
7616 %{
7617 predicate(CompressedOops::shift() == 0);
7618 match(Set dst mem);
7619
7620 ins_cost(110);
7621 format %{ "leaq $dst, $mem\t# ptr off32narrow" %}
7622 ins_encode %{
7623 __ leaq($dst$$Register, $mem$$Address);
7624 %}
7625 ins_pipe(ialu_reg_reg_fat);
7626 %}
7627
7628 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
7629 %{
7630 predicate(CompressedOops::shift() == 0);
7631 match(Set dst mem);
7632
7633 ins_cost(110);
7634 format %{ "leaq $dst, $mem\t# ptr idxoffnarrow" %}
7635 ins_encode %{
7636 __ leaq($dst$$Register, $mem$$Address);
7637 %}
7638 ins_pipe(ialu_reg_reg_fat);
7639 %}
7640
7641 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
7642 %{
7643 predicate(CompressedOops::shift() == 0);
7644 match(Set dst mem);
7645
7646 ins_cost(110);
7647 format %{ "leaq $dst, $mem\t# ptr idxscalenarrow" %}
7648 ins_encode %{
7649 __ leaq($dst$$Register, $mem$$Address);
7650 %}
7651 ins_pipe(ialu_reg_reg_fat);
7652 %}
7653
7654 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
7655 %{
7656 predicate(CompressedOops::shift() == 0);
7657 match(Set dst mem);
7658
7659 ins_cost(110);
7660 format %{ "leaq $dst, $mem\t# ptr idxscaleoffnarrow" %}
7661 ins_encode %{
7662 __ leaq($dst$$Register, $mem$$Address);
7663 %}
7664 ins_pipe(ialu_reg_reg_fat);
7665 %}
7666
7667 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
7668 %{
7669 predicate(CompressedOops::shift() == 0);
7670 match(Set dst mem);
7671
7672 ins_cost(110);
7673 format %{ "leaq $dst, $mem\t# ptr posidxoffnarrow" %}
7674 ins_encode %{
7675 __ leaq($dst$$Register, $mem$$Address);
7676 %}
7677 ins_pipe(ialu_reg_reg_fat);
7678 %}
7679
7680 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
7681 %{
7682 predicate(CompressedOops::shift() == 0);
7683 match(Set dst mem);
7684
7685 ins_cost(110);
7686 format %{ "leaq $dst, $mem\t# ptr posidxscaleoffnarrow" %}
7687 ins_encode %{
7688 __ leaq($dst$$Register, $mem$$Address);
7689 %}
7690 ins_pipe(ialu_reg_reg_fat);
7691 %}
7692
7693 instruct loadConI(rRegI dst, immI src)
7694 %{
7695 match(Set dst src);
7696
7697 format %{ "movl $dst, $src\t# int" %}
7698 ins_encode %{
7699 __ movl($dst$$Register, $src$$constant);
7700 %}
7701 ins_pipe(ialu_reg_fat); // XXX
7702 %}
7703
7704 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
7705 %{
7706 match(Set dst src);
7707 effect(KILL cr);
7708
7709 ins_cost(50);
7710 format %{ "xorl $dst, $dst\t# int" %}
7711 ins_encode %{
7712 __ xorl($dst$$Register, $dst$$Register);
7713 %}
7714 ins_pipe(ialu_reg);
7715 %}
7716
7717 instruct loadConL(rRegL dst, immL src)
7718 %{
7719 match(Set dst src);
7720
7721 ins_cost(150);
7722 format %{ "movq $dst, $src\t# long" %}
7723 ins_encode %{
7724 __ mov64($dst$$Register, $src$$constant);
7725 %}
7726 ins_pipe(ialu_reg);
7727 %}
7728
7729 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
7730 %{
7731 match(Set dst src);
7732 effect(KILL cr);
7733
7734 ins_cost(50);
7735 format %{ "xorl $dst, $dst\t# long" %}
7736 ins_encode %{
7737 __ xorl($dst$$Register, $dst$$Register);
7738 %}
7739 ins_pipe(ialu_reg); // XXX
7740 %}
7741
7742 instruct loadConUL32(rRegL dst, immUL32 src)
7743 %{
7744 match(Set dst src);
7745
7746 ins_cost(60);
7747 format %{ "movl $dst, $src\t# long (unsigned 32-bit)" %}
7748 ins_encode %{
7749 __ movl($dst$$Register, $src$$constant);
7750 %}
7751 ins_pipe(ialu_reg);
7752 %}
7753
7754 instruct loadConL32(rRegL dst, immL32 src)
7755 %{
7756 match(Set dst src);
7757
7758 ins_cost(70);
7759 format %{ "movq $dst, $src\t# long (32-bit)" %}
7760 ins_encode %{
7761 __ movq($dst$$Register, $src$$constant);
7762 %}
7763 ins_pipe(ialu_reg);
7764 %}
7765
7766 instruct loadConP(rRegP dst, immP con) %{
7767 match(Set dst con);
7768
7769 format %{ "movq $dst, $con\t# ptr" %}
7770 ins_encode %{
7771 __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
7772 %}
7773 ins_pipe(ialu_reg_fat); // XXX
7774 %}
7775
7776 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
7777 %{
7778 match(Set dst src);
7779 effect(KILL cr);
7780
7781 ins_cost(50);
7782 format %{ "xorl $dst, $dst\t# ptr" %}
7783 ins_encode %{
7784 __ xorl($dst$$Register, $dst$$Register);
7785 %}
7786 ins_pipe(ialu_reg);
7787 %}
7788
7789 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
7790 %{
7791 match(Set dst src);
7792 effect(KILL cr);
7793
7794 ins_cost(60);
7795 format %{ "movl $dst, $src\t# ptr (positive 32-bit)" %}
7796 ins_encode %{
7797 __ movl($dst$$Register, $src$$constant);
7798 %}
7799 ins_pipe(ialu_reg);
7800 %}
7801
7802 instruct loadConF(regF dst, immF con) %{
7803 match(Set dst con);
7804 ins_cost(125);
7805 format %{ "movss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
7806 ins_encode %{
7807 __ movflt($dst$$XMMRegister, $constantaddress($con));
7808 %}
7809 ins_pipe(pipe_slow);
7810 %}
7811
7812 instruct loadConH(regF dst, immH con) %{
7813 match(Set dst con);
7814 ins_cost(125);
7815 format %{ "movss $dst, [$constantaddress]\t# load from constant table: halffloat=$con" %}
7816 ins_encode %{
7817 __ movflt($dst$$XMMRegister, $constantaddress($con));
7818 %}
7819 ins_pipe(pipe_slow);
7820 %}
7821
7822 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
7823 match(Set dst src);
7824 effect(KILL cr);
7825 format %{ "xorq $dst, $src\t# compressed null pointer" %}
7826 ins_encode %{
7827 __ xorq($dst$$Register, $dst$$Register);
7828 %}
7829 ins_pipe(ialu_reg);
7830 %}
7831
7832 instruct loadConN(rRegN dst, immN src) %{
7833 match(Set dst src);
7834
7835 ins_cost(125);
7836 format %{ "movl $dst, $src\t# compressed ptr" %}
7837 ins_encode %{
7838 address con = (address)$src$$constant;
7839 if (con == nullptr) {
7840 ShouldNotReachHere();
7841 } else {
7842 __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
7843 }
7844 %}
7845 ins_pipe(ialu_reg_fat); // XXX
7846 %}
7847
7848 instruct loadConNKlass(rRegN dst, immNKlass src) %{
7849 match(Set dst src);
7850
7851 ins_cost(125);
7852 format %{ "movl $dst, $src\t# compressed klass ptr" %}
7853 ins_encode %{
7854 address con = (address)$src$$constant;
7855 if (con == nullptr) {
7856 ShouldNotReachHere();
7857 } else {
7858 __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
7859 }
7860 %}
7861 ins_pipe(ialu_reg_fat); // XXX
7862 %}
7863
7864 instruct loadConF0(regF dst, immF0 src)
7865 %{
7866 match(Set dst src);
7867 ins_cost(100);
7868
7869 format %{ "xorps $dst, $dst\t# float 0.0" %}
7870 ins_encode %{
7871 __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
7872 %}
7873 ins_pipe(pipe_slow);
7874 %}
7875
7876 // Use the same format since predicate() can not be used here.
7877 instruct loadConD(regD dst, immD con) %{
7878 match(Set dst con);
7879 ins_cost(125);
7880 format %{ "movsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
7881 ins_encode %{
7882 __ movdbl($dst$$XMMRegister, $constantaddress($con));
7883 %}
7884 ins_pipe(pipe_slow);
7885 %}
7886
7887 instruct loadConD0(regD dst, immD0 src)
7888 %{
7889 match(Set dst src);
7890 ins_cost(100);
7891
7892 format %{ "xorpd $dst, $dst\t# double 0.0" %}
7893 ins_encode %{
7894 __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
7895 %}
7896 ins_pipe(pipe_slow);
7897 %}
7898
7899 instruct loadSSI(rRegI dst, stackSlotI src)
7900 %{
7901 match(Set dst src);
7902
7903 ins_cost(125);
7904 format %{ "movl $dst, $src\t# int stk" %}
7905 ins_encode %{
7906 __ movl($dst$$Register, $src$$Address);
7907 %}
7908 ins_pipe(ialu_reg_mem);
7909 %}
7910
7911 instruct loadSSL(rRegL dst, stackSlotL src)
7912 %{
7913 match(Set dst src);
7914
7915 ins_cost(125);
7916 format %{ "movq $dst, $src\t# long stk" %}
7917 ins_encode %{
7918 __ movq($dst$$Register, $src$$Address);
7919 %}
7920 ins_pipe(ialu_reg_mem);
7921 %}
7922
7923 instruct loadSSP(rRegP dst, stackSlotP src)
7924 %{
7925 match(Set dst src);
7926
7927 ins_cost(125);
7928 format %{ "movq $dst, $src\t# ptr stk" %}
7929 ins_encode %{
7930 __ movq($dst$$Register, $src$$Address);
7931 %}
7932 ins_pipe(ialu_reg_mem);
7933 %}
7934
7935 instruct loadSSF(regF dst, stackSlotF src)
7936 %{
7937 match(Set dst src);
7938
7939 ins_cost(125);
7940 format %{ "movss $dst, $src\t# float stk" %}
7941 ins_encode %{
7942 __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
7943 %}
7944 ins_pipe(pipe_slow); // XXX
7945 %}
7946
7947 // Use the same format since predicate() can not be used here.
7948 instruct loadSSD(regD dst, stackSlotD src)
7949 %{
7950 match(Set dst src);
7951
7952 ins_cost(125);
7953 format %{ "movsd $dst, $src\t# double stk" %}
7954 ins_encode %{
7955 __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
7956 %}
7957 ins_pipe(pipe_slow); // XXX
7958 %}
7959
7960 // Prefetch instructions for allocation.
7961 // Must be safe to execute with invalid address (cannot fault).
7962
7963 instruct prefetchAlloc( memory mem ) %{
7964 predicate(AllocatePrefetchInstr==3);
7965 match(PrefetchAllocation mem);
7966 ins_cost(125);
7967
7968 format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
7969 ins_encode %{
7970 __ prefetchw($mem$$Address);
7971 %}
7972 ins_pipe(ialu_mem);
7973 %}
7974
7975 instruct prefetchAllocNTA( memory mem ) %{
7976 predicate(AllocatePrefetchInstr==0);
7977 match(PrefetchAllocation mem);
7978 ins_cost(125);
7979
7980 format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
7981 ins_encode %{
7982 __ prefetchnta($mem$$Address);
7983 %}
7984 ins_pipe(ialu_mem);
7985 %}
7986
7987 instruct prefetchAllocT0( memory mem ) %{
7988 predicate(AllocatePrefetchInstr==1);
7989 match(PrefetchAllocation mem);
7990 ins_cost(125);
7991
7992 format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
7993 ins_encode %{
7994 __ prefetcht0($mem$$Address);
7995 %}
7996 ins_pipe(ialu_mem);
7997 %}
7998
7999 instruct prefetchAllocT2( memory mem ) %{
8000 predicate(AllocatePrefetchInstr==2);
8001 match(PrefetchAllocation mem);
8002 ins_cost(125);
8003
8004 format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
8005 ins_encode %{
8006 __ prefetcht2($mem$$Address);
8007 %}
8008 ins_pipe(ialu_mem);
8009 %}
8010
8011 //----------Store Instructions-------------------------------------------------
8012
8013 // Store Byte
8014 instruct storeB(memory mem, rRegI src)
8015 %{
8016 match(Set mem (StoreB mem src));
8017
8018 ins_cost(125); // XXX
8019 format %{ "movb $mem, $src\t# byte" %}
8020 ins_encode %{
8021 __ movb($mem$$Address, $src$$Register);
8022 %}
8023 ins_pipe(ialu_mem_reg);
8024 %}
8025
8026 // Store Char/Short
8027 instruct storeC(memory mem, rRegI src)
8028 %{
8029 match(Set mem (StoreC mem src));
8030
8031 ins_cost(125); // XXX
8032 format %{ "movw $mem, $src\t# char/short" %}
8033 ins_encode %{
8034 __ movw($mem$$Address, $src$$Register);
8035 %}
8036 ins_pipe(ialu_mem_reg);
8037 %}
8038
8039 // Store Integer
8040 instruct storeI(memory mem, rRegI src)
8041 %{
8042 match(Set mem (StoreI mem src));
8043
8044 ins_cost(125); // XXX
8045 format %{ "movl $mem, $src\t# int" %}
8046 ins_encode %{
8047 __ movl($mem$$Address, $src$$Register);
8048 %}
8049 ins_pipe(ialu_mem_reg);
8050 %}
8051
8052 // Store Long
8053 instruct storeL(memory mem, rRegL src)
8054 %{
8055 match(Set mem (StoreL mem src));
8056
8057 ins_cost(125); // XXX
8058 format %{ "movq $mem, $src\t# long" %}
8059 ins_encode %{
8060 __ movq($mem$$Address, $src$$Register);
8061 %}
8062 ins_pipe(ialu_mem_reg); // XXX
8063 %}
8064
8065 // Store Pointer
8066 instruct storeP(memory mem, any_RegP src)
8067 %{
8068 predicate(n->as_Store()->barrier_data() == 0);
8069 match(Set mem (StoreP mem src));
8070
8071 ins_cost(125); // XXX
8072 format %{ "movq $mem, $src\t# ptr" %}
8073 ins_encode %{
8074 __ movq($mem$$Address, $src$$Register);
8075 %}
8076 ins_pipe(ialu_mem_reg);
8077 %}
8078
8079 instruct storeImmP0(memory mem, immP0 zero)
8080 %{
8081 predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
8082 match(Set mem (StoreP mem zero));
8083
8084 ins_cost(125); // XXX
8085 format %{ "movq $mem, R12\t# ptr (R12_heapbase==0)" %}
8086 ins_encode %{
8087 __ movq($mem$$Address, r12);
8088 %}
8089 ins_pipe(ialu_mem_reg);
8090 %}
8091
8092 // Store Null Pointer, mark word, or other simple pointer constant.
8093 instruct storeImmP(memory mem, immP31 src)
8094 %{
8095 predicate(n->as_Store()->barrier_data() == 0);
8096 match(Set mem (StoreP mem src));
8097
8098 ins_cost(150); // XXX
8099 format %{ "movq $mem, $src\t# ptr" %}
8100 ins_encode %{
8101 __ movq($mem$$Address, $src$$constant);
8102 %}
8103 ins_pipe(ialu_mem_imm);
8104 %}
8105
8106 // Store Compressed Pointer
8107 instruct storeN(memory mem, rRegN src)
8108 %{
8109 predicate(n->as_Store()->barrier_data() == 0);
8110 match(Set mem (StoreN mem src));
8111
8112 ins_cost(125); // XXX
8113 format %{ "movl $mem, $src\t# compressed ptr" %}
8114 ins_encode %{
8115 __ movl($mem$$Address, $src$$Register);
8116 %}
8117 ins_pipe(ialu_mem_reg);
8118 %}
8119
8120 instruct storeNKlass(memory mem, rRegN src)
8121 %{
8122 match(Set mem (StoreNKlass mem src));
8123
8124 ins_cost(125); // XXX
8125 format %{ "movl $mem, $src\t# compressed klass ptr" %}
8126 ins_encode %{
8127 __ movl($mem$$Address, $src$$Register);
8128 %}
8129 ins_pipe(ialu_mem_reg);
8130 %}
8131
8132 instruct storeImmN0(memory mem, immN0 zero)
8133 %{
8134 predicate(CompressedOops::base() == nullptr && n->as_Store()->barrier_data() == 0);
8135 match(Set mem (StoreN mem zero));
8136
8137 ins_cost(125); // XXX
8138 format %{ "movl $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
8139 ins_encode %{
8140 __ movl($mem$$Address, r12);
8141 %}
8142 ins_pipe(ialu_mem_reg);
8143 %}
8144
8145 instruct storeImmN(memory mem, immN src)
8146 %{
8147 predicate(n->as_Store()->barrier_data() == 0);
8148 match(Set mem (StoreN mem src));
8149
8150 ins_cost(150); // XXX
8151 format %{ "movl $mem, $src\t# compressed ptr" %}
8152 ins_encode %{
8153 address con = (address)$src$$constant;
8154 if (con == nullptr) {
8155 __ movl($mem$$Address, 0);
8156 } else {
8157 __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
8158 }
8159 %}
8160 ins_pipe(ialu_mem_imm);
8161 %}
8162
8163 instruct storeImmNKlass(memory mem, immNKlass src)
8164 %{
8165 match(Set mem (StoreNKlass mem src));
8166
8167 ins_cost(150); // XXX
8168 format %{ "movl $mem, $src\t# compressed klass ptr" %}
8169 ins_encode %{
8170 __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
8171 %}
8172 ins_pipe(ialu_mem_imm);
8173 %}
8174
8175 // Store Integer Immediate
8176 instruct storeImmI0(memory mem, immI_0 zero)
8177 %{
8178 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8179 match(Set mem (StoreI mem zero));
8180
8181 ins_cost(125); // XXX
8182 format %{ "movl $mem, R12\t# int (R12_heapbase==0)" %}
8183 ins_encode %{
8184 __ movl($mem$$Address, r12);
8185 %}
8186 ins_pipe(ialu_mem_reg);
8187 %}
8188
8189 instruct storeImmI(memory mem, immI src)
8190 %{
8191 match(Set mem (StoreI mem src));
8192
8193 ins_cost(150);
8194 format %{ "movl $mem, $src\t# int" %}
8195 ins_encode %{
8196 __ movl($mem$$Address, $src$$constant);
8197 %}
8198 ins_pipe(ialu_mem_imm);
8199 %}
8200
8201 // Store Long Immediate
8202 instruct storeImmL0(memory mem, immL0 zero)
8203 %{
8204 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8205 match(Set mem (StoreL mem zero));
8206
8207 ins_cost(125); // XXX
8208 format %{ "movq $mem, R12\t# long (R12_heapbase==0)" %}
8209 ins_encode %{
8210 __ movq($mem$$Address, r12);
8211 %}
8212 ins_pipe(ialu_mem_reg);
8213 %}
8214
8215 instruct storeImmL(memory mem, immL32 src)
8216 %{
8217 match(Set mem (StoreL mem src));
8218
8219 ins_cost(150);
8220 format %{ "movq $mem, $src\t# long" %}
8221 ins_encode %{
8222 __ movq($mem$$Address, $src$$constant);
8223 %}
8224 ins_pipe(ialu_mem_imm);
8225 %}
8226
8227 // Store Short/Char Immediate
8228 instruct storeImmC0(memory mem, immI_0 zero)
8229 %{
8230 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8231 match(Set mem (StoreC mem zero));
8232
8233 ins_cost(125); // XXX
8234 format %{ "movw $mem, R12\t# short/char (R12_heapbase==0)" %}
8235 ins_encode %{
8236 __ movw($mem$$Address, r12);
8237 %}
8238 ins_pipe(ialu_mem_reg);
8239 %}
8240
8241 instruct storeImmI16(memory mem, immI16 src)
8242 %{
8243 predicate(UseStoreImmI16);
8244 match(Set mem (StoreC mem src));
8245
8246 ins_cost(150);
8247 format %{ "movw $mem, $src\t# short/char" %}
8248 ins_encode %{
8249 __ movw($mem$$Address, $src$$constant);
8250 %}
8251 ins_pipe(ialu_mem_imm);
8252 %}
8253
8254 // Store Byte Immediate
8255 instruct storeImmB0(memory mem, immI_0 zero)
8256 %{
8257 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8258 match(Set mem (StoreB mem zero));
8259
8260 ins_cost(125); // XXX
8261 format %{ "movb $mem, R12\t# short/char (R12_heapbase==0)" %}
8262 ins_encode %{
8263 __ movb($mem$$Address, r12);
8264 %}
8265 ins_pipe(ialu_mem_reg);
8266 %}
8267
8268 instruct storeImmB(memory mem, immI8 src)
8269 %{
8270 match(Set mem (StoreB mem src));
8271
8272 ins_cost(150); // XXX
8273 format %{ "movb $mem, $src\t# byte" %}
8274 ins_encode %{
8275 __ movb($mem$$Address, $src$$constant);
8276 %}
8277 ins_pipe(ialu_mem_imm);
8278 %}
8279
8280 // Store Float
8281 instruct storeF(memory mem, regF src)
8282 %{
8283 match(Set mem (StoreF mem src));
8284
8285 ins_cost(95); // XXX
8286 format %{ "movss $mem, $src\t# float" %}
8287 ins_encode %{
8288 __ movflt($mem$$Address, $src$$XMMRegister);
8289 %}
8290 ins_pipe(pipe_slow); // XXX
8291 %}
8292
8293 // Store immediate Float value (it is faster than store from XMM register)
8294 instruct storeF0(memory mem, immF0 zero)
8295 %{
8296 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8297 match(Set mem (StoreF mem zero));
8298
8299 ins_cost(25); // XXX
8300 format %{ "movl $mem, R12\t# float 0. (R12_heapbase==0)" %}
8301 ins_encode %{
8302 __ movl($mem$$Address, r12);
8303 %}
8304 ins_pipe(ialu_mem_reg);
8305 %}
8306
8307 instruct storeF_imm(memory mem, immF src)
8308 %{
8309 match(Set mem (StoreF mem src));
8310
8311 ins_cost(50);
8312 format %{ "movl $mem, $src\t# float" %}
8313 ins_encode %{
8314 __ movl($mem$$Address, jint_cast($src$$constant));
8315 %}
8316 ins_pipe(ialu_mem_imm);
8317 %}
8318
8319 // Store Double
8320 instruct storeD(memory mem, regD src)
8321 %{
8322 match(Set mem (StoreD mem src));
8323
8324 ins_cost(95); // XXX
8325 format %{ "movsd $mem, $src\t# double" %}
8326 ins_encode %{
8327 __ movdbl($mem$$Address, $src$$XMMRegister);
8328 %}
8329 ins_pipe(pipe_slow); // XXX
8330 %}
8331
8332 // Store immediate double 0.0 (it is faster than store from XMM register)
8333 instruct storeD0_imm(memory mem, immD0 src)
8334 %{
8335 predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
8336 match(Set mem (StoreD mem src));
8337
8338 ins_cost(50);
8339 format %{ "movq $mem, $src\t# double 0." %}
8340 ins_encode %{
8341 __ movq($mem$$Address, $src$$constant);
8342 %}
8343 ins_pipe(ialu_mem_imm);
8344 %}
8345
8346 instruct storeD0(memory mem, immD0 zero)
8347 %{
8348 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8349 match(Set mem (StoreD mem zero));
8350
8351 ins_cost(25); // XXX
8352 format %{ "movq $mem, R12\t# double 0. (R12_heapbase==0)" %}
8353 ins_encode %{
8354 __ movq($mem$$Address, r12);
8355 %}
8356 ins_pipe(ialu_mem_reg);
8357 %}
8358
8359 instruct storeSSI(stackSlotI dst, rRegI src)
8360 %{
8361 match(Set dst src);
8362
8363 ins_cost(100);
8364 format %{ "movl $dst, $src\t# int stk" %}
8365 ins_encode %{
8366 __ movl($dst$$Address, $src$$Register);
8367 %}
8368 ins_pipe( ialu_mem_reg );
8369 %}
8370
8371 instruct storeSSL(stackSlotL dst, rRegL src)
8372 %{
8373 match(Set dst src);
8374
8375 ins_cost(100);
8376 format %{ "movq $dst, $src\t# long stk" %}
8377 ins_encode %{
8378 __ movq($dst$$Address, $src$$Register);
8379 %}
8380 ins_pipe(ialu_mem_reg);
8381 %}
8382
8383 instruct storeSSP(stackSlotP dst, rRegP src)
8384 %{
8385 match(Set dst src);
8386
8387 ins_cost(100);
8388 format %{ "movq $dst, $src\t# ptr stk" %}
8389 ins_encode %{
8390 __ movq($dst$$Address, $src$$Register);
8391 %}
8392 ins_pipe(ialu_mem_reg);
8393 %}
8394
8395 instruct storeSSF(stackSlotF dst, regF src)
8396 %{
8397 match(Set dst src);
8398
8399 ins_cost(95); // XXX
8400 format %{ "movss $dst, $src\t# float stk" %}
8401 ins_encode %{
8402 __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
8403 %}
8404 ins_pipe(pipe_slow); // XXX
8405 %}
8406
8407 instruct storeSSD(stackSlotD dst, regD src)
8408 %{
8409 match(Set dst src);
8410
8411 ins_cost(95); // XXX
8412 format %{ "movsd $dst, $src\t# double stk" %}
8413 ins_encode %{
8414 __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
8415 %}
8416 ins_pipe(pipe_slow); // XXX
8417 %}
8418
8419 instruct cacheWB(indirect addr)
8420 %{
8421 predicate(VM_Version::supports_data_cache_line_flush());
8422 match(CacheWB addr);
8423
8424 ins_cost(100);
8425 format %{"cache wb $addr" %}
8426 ins_encode %{
8427 assert($addr->index_position() < 0, "should be");
8428 assert($addr$$disp == 0, "should be");
8429 __ cache_wb(Address($addr$$base$$Register, 0));
8430 %}
8431 ins_pipe(pipe_slow); // XXX
8432 %}
8433
8434 instruct cacheWBPreSync()
8435 %{
8436 predicate(VM_Version::supports_data_cache_line_flush());
8437 match(CacheWBPreSync);
8438
8439 ins_cost(100);
8440 format %{"cache wb presync" %}
8441 ins_encode %{
8442 __ cache_wbsync(true);
8443 %}
8444 ins_pipe(pipe_slow); // XXX
8445 %}
8446
8447 instruct cacheWBPostSync()
8448 %{
8449 predicate(VM_Version::supports_data_cache_line_flush());
8450 match(CacheWBPostSync);
8451
8452 ins_cost(100);
8453 format %{"cache wb postsync" %}
8454 ins_encode %{
8455 __ cache_wbsync(false);
8456 %}
8457 ins_pipe(pipe_slow); // XXX
8458 %}
8459
8460 //----------BSWAP Instructions-------------------------------------------------
8461 instruct bytes_reverse_int(rRegI dst) %{
8462 match(Set dst (ReverseBytesI dst));
8463
8464 format %{ "bswapl $dst" %}
8465 ins_encode %{
8466 __ bswapl($dst$$Register);
8467 %}
8468 ins_pipe( ialu_reg );
8469 %}
8470
8471 instruct bytes_reverse_long(rRegL dst) %{
8472 match(Set dst (ReverseBytesL dst));
8473
8474 format %{ "bswapq $dst" %}
8475 ins_encode %{
8476 __ bswapq($dst$$Register);
8477 %}
8478 ins_pipe( ialu_reg);
8479 %}
8480
8481 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
8482 match(Set dst (ReverseBytesUS dst));
8483 effect(KILL cr);
8484
8485 format %{ "bswapl $dst\n\t"
8486 "shrl $dst,16\n\t" %}
8487 ins_encode %{
8488 __ bswapl($dst$$Register);
8489 __ shrl($dst$$Register, 16);
8490 %}
8491 ins_pipe( ialu_reg );
8492 %}
8493
8494 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
8495 match(Set dst (ReverseBytesS dst));
8496 effect(KILL cr);
8497
8498 format %{ "bswapl $dst\n\t"
8499 "sar $dst,16\n\t" %}
8500 ins_encode %{
8501 __ bswapl($dst$$Register);
8502 __ sarl($dst$$Register, 16);
8503 %}
8504 ins_pipe( ialu_reg );
8505 %}
8506
8507 //---------- Zeros Count Instructions ------------------------------------------
8508
8509 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
8510 predicate(UseCountLeadingZerosInstruction);
8511 match(Set dst (CountLeadingZerosI src));
8512 effect(KILL cr);
8513
8514 format %{ "lzcntl $dst, $src\t# count leading zeros (int)" %}
8515 ins_encode %{
8516 __ lzcntl($dst$$Register, $src$$Register);
8517 %}
8518 ins_pipe(ialu_reg);
8519 %}
8520
8521 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
8522 predicate(UseCountLeadingZerosInstruction);
8523 match(Set dst (CountLeadingZerosI (LoadI src)));
8524 effect(KILL cr);
8525 ins_cost(175);
8526 format %{ "lzcntl $dst, $src\t# count leading zeros (int)" %}
8527 ins_encode %{
8528 __ lzcntl($dst$$Register, $src$$Address);
8529 %}
8530 ins_pipe(ialu_reg_mem);
8531 %}
8532
8533 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
8534 predicate(!UseCountLeadingZerosInstruction);
8535 match(Set dst (CountLeadingZerosI src));
8536 effect(KILL cr);
8537
8538 format %{ "bsrl $dst, $src\t# count leading zeros (int)\n\t"
8539 "jnz skip\n\t"
8540 "movl $dst, -1\n"
8541 "skip:\n\t"
8542 "negl $dst\n\t"
8543 "addl $dst, 31" %}
8544 ins_encode %{
8545 Register Rdst = $dst$$Register;
8546 Register Rsrc = $src$$Register;
8547 Label skip;
8548 __ bsrl(Rdst, Rsrc);
8549 __ jccb(Assembler::notZero, skip);
8550 __ movl(Rdst, -1);
8551 __ bind(skip);
8552 __ negl(Rdst);
8553 __ addl(Rdst, BitsPerInt - 1);
8554 %}
8555 ins_pipe(ialu_reg);
8556 %}
8557
8558 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
8559 predicate(UseCountLeadingZerosInstruction);
8560 match(Set dst (CountLeadingZerosL src));
8561 effect(KILL cr);
8562
8563 format %{ "lzcntq $dst, $src\t# count leading zeros (long)" %}
8564 ins_encode %{
8565 __ lzcntq($dst$$Register, $src$$Register);
8566 %}
8567 ins_pipe(ialu_reg);
8568 %}
8569
8570 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
8571 predicate(UseCountLeadingZerosInstruction);
8572 match(Set dst (CountLeadingZerosL (LoadL src)));
8573 effect(KILL cr);
8574 ins_cost(175);
8575 format %{ "lzcntq $dst, $src\t# count leading zeros (long)" %}
8576 ins_encode %{
8577 __ lzcntq($dst$$Register, $src$$Address);
8578 %}
8579 ins_pipe(ialu_reg_mem);
8580 %}
8581
8582 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
8583 predicate(!UseCountLeadingZerosInstruction);
8584 match(Set dst (CountLeadingZerosL src));
8585 effect(KILL cr);
8586
8587 format %{ "bsrq $dst, $src\t# count leading zeros (long)\n\t"
8588 "jnz skip\n\t"
8589 "movl $dst, -1\n"
8590 "skip:\n\t"
8591 "negl $dst\n\t"
8592 "addl $dst, 63" %}
8593 ins_encode %{
8594 Register Rdst = $dst$$Register;
8595 Register Rsrc = $src$$Register;
8596 Label skip;
8597 __ bsrq(Rdst, Rsrc);
8598 __ jccb(Assembler::notZero, skip);
8599 __ movl(Rdst, -1);
8600 __ bind(skip);
8601 __ negl(Rdst);
8602 __ addl(Rdst, BitsPerLong - 1);
8603 %}
8604 ins_pipe(ialu_reg);
8605 %}
8606
8607 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
8608 predicate(UseCountTrailingZerosInstruction);
8609 match(Set dst (CountTrailingZerosI src));
8610 effect(KILL cr);
8611
8612 format %{ "tzcntl $dst, $src\t# count trailing zeros (int)" %}
8613 ins_encode %{
8614 __ tzcntl($dst$$Register, $src$$Register);
8615 %}
8616 ins_pipe(ialu_reg);
8617 %}
8618
8619 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
8620 predicate(UseCountTrailingZerosInstruction);
8621 match(Set dst (CountTrailingZerosI (LoadI src)));
8622 effect(KILL cr);
8623 ins_cost(175);
8624 format %{ "tzcntl $dst, $src\t# count trailing zeros (int)" %}
8625 ins_encode %{
8626 __ tzcntl($dst$$Register, $src$$Address);
8627 %}
8628 ins_pipe(ialu_reg_mem);
8629 %}
8630
8631 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
8632 predicate(!UseCountTrailingZerosInstruction);
8633 match(Set dst (CountTrailingZerosI src));
8634 effect(KILL cr);
8635
8636 format %{ "bsfl $dst, $src\t# count trailing zeros (int)\n\t"
8637 "jnz done\n\t"
8638 "movl $dst, 32\n"
8639 "done:" %}
8640 ins_encode %{
8641 Register Rdst = $dst$$Register;
8642 Label done;
8643 __ bsfl(Rdst, $src$$Register);
8644 __ jccb(Assembler::notZero, done);
8645 __ movl(Rdst, BitsPerInt);
8646 __ bind(done);
8647 %}
8648 ins_pipe(ialu_reg);
8649 %}
8650
8651 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
8652 predicate(UseCountTrailingZerosInstruction);
8653 match(Set dst (CountTrailingZerosL src));
8654 effect(KILL cr);
8655
8656 format %{ "tzcntq $dst, $src\t# count trailing zeros (long)" %}
8657 ins_encode %{
8658 __ tzcntq($dst$$Register, $src$$Register);
8659 %}
8660 ins_pipe(ialu_reg);
8661 %}
8662
8663 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
8664 predicate(UseCountTrailingZerosInstruction);
8665 match(Set dst (CountTrailingZerosL (LoadL src)));
8666 effect(KILL cr);
8667 ins_cost(175);
8668 format %{ "tzcntq $dst, $src\t# count trailing zeros (long)" %}
8669 ins_encode %{
8670 __ tzcntq($dst$$Register, $src$$Address);
8671 %}
8672 ins_pipe(ialu_reg_mem);
8673 %}
8674
8675 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
8676 predicate(!UseCountTrailingZerosInstruction);
8677 match(Set dst (CountTrailingZerosL src));
8678 effect(KILL cr);
8679
8680 format %{ "bsfq $dst, $src\t# count trailing zeros (long)\n\t"
8681 "jnz done\n\t"
8682 "movl $dst, 64\n"
8683 "done:" %}
8684 ins_encode %{
8685 Register Rdst = $dst$$Register;
8686 Label done;
8687 __ bsfq(Rdst, $src$$Register);
8688 __ jccb(Assembler::notZero, done);
8689 __ movl(Rdst, BitsPerLong);
8690 __ bind(done);
8691 %}
8692 ins_pipe(ialu_reg);
8693 %}
8694
8695 //--------------- Reverse Operation Instructions ----------------
8696 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
8697 predicate(!VM_Version::supports_gfni());
8698 match(Set dst (ReverseI src));
8699 effect(TEMP dst, TEMP rtmp, KILL cr);
8700 format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
8701 ins_encode %{
8702 __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
8703 %}
8704 ins_pipe( ialu_reg );
8705 %}
8706
8707 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, vlRegF xtmp1, vlRegF xtmp2, rRegL rtmp, rFlagsReg cr) %{
8708 predicate(VM_Version::supports_gfni());
8709 match(Set dst (ReverseI src));
8710 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
8711 format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
8712 ins_encode %{
8713 __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
8714 %}
8715 ins_pipe( ialu_reg );
8716 %}
8717
8718 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
8719 predicate(!VM_Version::supports_gfni());
8720 match(Set dst (ReverseL src));
8721 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
8722 format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
8723 ins_encode %{
8724 __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
8725 %}
8726 ins_pipe( ialu_reg );
8727 %}
8728
8729 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, vlRegD xtmp1, vlRegD xtmp2, rRegL rtmp, rFlagsReg cr) %{
8730 predicate(VM_Version::supports_gfni());
8731 match(Set dst (ReverseL src));
8732 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
8733 format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
8734 ins_encode %{
8735 __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
8736 %}
8737 ins_pipe( ialu_reg );
8738 %}
8739
8740 //---------- Population Count Instructions -------------------------------------
8741
8742 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
8743 predicate(UsePopCountInstruction);
8744 match(Set dst (PopCountI src));
8745 effect(KILL cr);
8746
8747 format %{ "popcnt $dst, $src" %}
8748 ins_encode %{
8749 __ popcntl($dst$$Register, $src$$Register);
8750 %}
8751 ins_pipe(ialu_reg);
8752 %}
8753
8754 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
8755 predicate(UsePopCountInstruction);
8756 match(Set dst (PopCountI (LoadI mem)));
8757 effect(KILL cr);
8758
8759 format %{ "popcnt $dst, $mem" %}
8760 ins_encode %{
8761 __ popcntl($dst$$Register, $mem$$Address);
8762 %}
8763 ins_pipe(ialu_reg);
8764 %}
8765
8766 // Note: Long.bitCount(long) returns an int.
8767 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
8768 predicate(UsePopCountInstruction);
8769 match(Set dst (PopCountL src));
8770 effect(KILL cr);
8771
8772 format %{ "popcnt $dst, $src" %}
8773 ins_encode %{
8774 __ popcntq($dst$$Register, $src$$Register);
8775 %}
8776 ins_pipe(ialu_reg);
8777 %}
8778
8779 // Note: Long.bitCount(long) returns an int.
8780 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
8781 predicate(UsePopCountInstruction);
8782 match(Set dst (PopCountL (LoadL mem)));
8783 effect(KILL cr);
8784
8785 format %{ "popcnt $dst, $mem" %}
8786 ins_encode %{
8787 __ popcntq($dst$$Register, $mem$$Address);
8788 %}
8789 ins_pipe(ialu_reg);
8790 %}
8791
8792
8793 //----------MemBar Instructions-----------------------------------------------
8794 // Memory barrier flavors
8795
8796 instruct membar_acquire()
8797 %{
8798 match(MemBarAcquire);
8799 match(LoadFence);
8800 ins_cost(0);
8801
8802 size(0);
8803 format %{ "MEMBAR-acquire ! (empty encoding)" %}
8804 ins_encode();
8805 ins_pipe(empty);
8806 %}
8807
8808 instruct membar_acquire_lock()
8809 %{
8810 match(MemBarAcquireLock);
8811 ins_cost(0);
8812
8813 size(0);
8814 format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
8815 ins_encode();
8816 ins_pipe(empty);
8817 %}
8818
8819 instruct membar_release()
8820 %{
8821 match(MemBarRelease);
8822 match(StoreFence);
8823 ins_cost(0);
8824
8825 size(0);
8826 format %{ "MEMBAR-release ! (empty encoding)" %}
8827 ins_encode();
8828 ins_pipe(empty);
8829 %}
8830
8831 instruct membar_release_lock()
8832 %{
8833 match(MemBarReleaseLock);
8834 ins_cost(0);
8835
8836 size(0);
8837 format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
8838 ins_encode();
8839 ins_pipe(empty);
8840 %}
8841
8842 instruct membar_storeload(rFlagsReg cr) %{
8843 match(MemBarStoreLoad);
8844 effect(KILL cr);
8845 ins_cost(400);
8846
8847 format %{
8848 $$template
8849 $$emit$$"lock addl [rsp + #0], 0\t! membar_storeload"
8850 %}
8851 ins_encode %{
8852 __ membar(Assembler::StoreLoad);
8853 %}
8854 ins_pipe(pipe_slow);
8855 %}
8856
8857 instruct membar_volatile(rFlagsReg cr) %{
8858 match(MemBarVolatile);
8859 effect(KILL cr);
8860 ins_cost(400);
8861
8862 format %{
8863 $$template
8864 $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
8865 %}
8866 ins_encode %{
8867 __ membar(Assembler::StoreLoad);
8868 %}
8869 ins_pipe(pipe_slow);
8870 %}
8871
8872 instruct unnecessary_membar_volatile()
8873 %{
8874 match(MemBarVolatile);
8875 predicate(Matcher::post_store_load_barrier(n));
8876 ins_cost(0);
8877
8878 size(0);
8879 format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
8880 ins_encode();
8881 ins_pipe(empty);
8882 %}
8883
8884 instruct membar_full(rFlagsReg cr) %{
8885 match(MemBarFull);
8886 effect(KILL cr);
8887 ins_cost(400);
8888
8889 format %{
8890 $$template
8891 $$emit$$"lock addl [rsp + #0], 0\t! membar_full"
8892 %}
8893 ins_encode %{
8894 __ membar(Assembler::StoreLoad);
8895 %}
8896 ins_pipe(pipe_slow);
8897 %}
8898
8899 instruct membar_storestore() %{
8900 match(MemBarStoreStore);
8901 match(StoreStoreFence);
8902 ins_cost(0);
8903
8904 size(0);
8905 format %{ "MEMBAR-storestore (empty encoding)" %}
8906 ins_encode( );
8907 ins_pipe(empty);
8908 %}
8909
8910 //----------Move Instructions--------------------------------------------------
8911
8912 instruct castX2P(rRegP dst, rRegL src)
8913 %{
8914 match(Set dst (CastX2P src));
8915
8916 format %{ "movq $dst, $src\t# long->ptr" %}
8917 ins_encode %{
8918 if ($dst$$reg != $src$$reg) {
8919 __ movptr($dst$$Register, $src$$Register);
8920 }
8921 %}
8922 ins_pipe(ialu_reg_reg); // XXX
8923 %}
8924
8925 instruct castP2X(rRegL dst, rRegP src)
8926 %{
8927 match(Set dst (CastP2X src));
8928
8929 format %{ "movq $dst, $src\t# ptr -> long" %}
8930 ins_encode %{
8931 if ($dst$$reg != $src$$reg) {
8932 __ movptr($dst$$Register, $src$$Register);
8933 }
8934 %}
8935 ins_pipe(ialu_reg_reg); // XXX
8936 %}
8937
8938 // Convert oop into int for vectors alignment masking
8939 instruct convP2I(rRegI dst, rRegP src)
8940 %{
8941 match(Set dst (ConvL2I (CastP2X src)));
8942
8943 format %{ "movl $dst, $src\t# ptr -> int" %}
8944 ins_encode %{
8945 __ movl($dst$$Register, $src$$Register);
8946 %}
8947 ins_pipe(ialu_reg_reg); // XXX
8948 %}
8949
8950 // Convert compressed oop into int for vectors alignment masking
8951 // in case of 32bit oops (heap < 4Gb).
8952 instruct convN2I(rRegI dst, rRegN src)
8953 %{
8954 predicate(CompressedOops::shift() == 0);
8955 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8956
8957 format %{ "movl $dst, $src\t# compressed ptr -> int" %}
8958 ins_encode %{
8959 __ movl($dst$$Register, $src$$Register);
8960 %}
8961 ins_pipe(ialu_reg_reg); // XXX
8962 %}
8963
8964 // Convert oop pointer into compressed form
8965 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
8966 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8967 match(Set dst (EncodeP src));
8968 effect(KILL cr);
8969 format %{ "encode_heap_oop $dst,$src" %}
8970 ins_encode %{
8971 Register s = $src$$Register;
8972 Register d = $dst$$Register;
8973 if (s != d) {
8974 __ movq(d, s);
8975 }
8976 __ encode_heap_oop(d);
8977 %}
8978 ins_pipe(ialu_reg_long);
8979 %}
8980
8981 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
8982 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8983 match(Set dst (EncodeP src));
8984 effect(KILL cr);
8985 format %{ "encode_heap_oop_not_null $dst,$src" %}
8986 ins_encode %{
8987 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8988 %}
8989 ins_pipe(ialu_reg_long);
8990 %}
8991
8992 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
8993 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8994 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8995 match(Set dst (DecodeN src));
8996 effect(KILL cr);
8997 format %{ "decode_heap_oop $dst,$src" %}
8998 ins_encode %{
8999 Register s = $src$$Register;
9000 Register d = $dst$$Register;
9001 if (s != d) {
9002 __ movq(d, s);
9003 }
9004 __ decode_heap_oop(d);
9005 %}
9006 ins_pipe(ialu_reg_long);
9007 %}
9008
9009 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
9010 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
9011 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
9012 match(Set dst (DecodeN src));
9013 effect(KILL cr);
9014 format %{ "decode_heap_oop_not_null $dst,$src" %}
9015 ins_encode %{
9016 Register s = $src$$Register;
9017 Register d = $dst$$Register;
9018 if (s != d) {
9019 __ decode_heap_oop_not_null(d, s);
9020 } else {
9021 __ decode_heap_oop_not_null(d);
9022 }
9023 %}
9024 ins_pipe(ialu_reg_long);
9025 %}
9026
9027 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
9028 match(Set dst (EncodePKlass src));
9029 effect(TEMP dst, KILL cr);
9030 format %{ "encode_and_move_klass_not_null $dst,$src" %}
9031 ins_encode %{
9032 __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
9033 %}
9034 ins_pipe(ialu_reg_long);
9035 %}
9036
9037 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
9038 match(Set dst (DecodeNKlass src));
9039 effect(TEMP dst, KILL cr);
9040 format %{ "decode_and_move_klass_not_null $dst,$src" %}
9041 ins_encode %{
9042 __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
9043 %}
9044 ins_pipe(ialu_reg_long);
9045 %}
9046
9047 //----------Conditional Move---------------------------------------------------
9048 // Jump
9049 // dummy instruction for generating temp registers
9050 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
9051 match(Jump (LShiftL switch_val shift));
9052 ins_cost(350);
9053 predicate(false);
9054 effect(TEMP dest);
9055
9056 format %{ "leaq $dest, [$constantaddress]\n\t"
9057 "jmp [$dest + $switch_val << $shift]\n\t" %}
9058 ins_encode %{
9059 // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
9060 // to do that and the compiler is using that register as one it can allocate.
9061 // So we build it all by hand.
9062 // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
9063 // ArrayAddress dispatch(table, index);
9064 Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
9065 __ lea($dest$$Register, $constantaddress);
9066 __ jmp(dispatch);
9067 %}
9068 ins_pipe(pipe_jmp);
9069 %}
9070
9071 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
9072 match(Jump (AddL (LShiftL switch_val shift) offset));
9073 ins_cost(350);
9074 effect(TEMP dest);
9075
9076 format %{ "leaq $dest, [$constantaddress]\n\t"
9077 "jmp [$dest + $switch_val << $shift + $offset]\n\t" %}
9078 ins_encode %{
9079 // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
9080 // to do that and the compiler is using that register as one it can allocate.
9081 // So we build it all by hand.
9082 // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
9083 // ArrayAddress dispatch(table, index);
9084 Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
9085 __ lea($dest$$Register, $constantaddress);
9086 __ jmp(dispatch);
9087 %}
9088 ins_pipe(pipe_jmp);
9089 %}
9090
9091 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
9092 match(Jump switch_val);
9093 ins_cost(350);
9094 effect(TEMP dest);
9095
9096 format %{ "leaq $dest, [$constantaddress]\n\t"
9097 "jmp [$dest + $switch_val]\n\t" %}
9098 ins_encode %{
9099 // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
9100 // to do that and the compiler is using that register as one it can allocate.
9101 // So we build it all by hand.
9102 // Address index(noreg, switch_reg, Address::times_1);
9103 // ArrayAddress dispatch(table, index);
9104 Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
9105 __ lea($dest$$Register, $constantaddress);
9106 __ jmp(dispatch);
9107 %}
9108 ins_pipe(pipe_jmp);
9109 %}
9110
9111 // Conditional move
9112 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
9113 %{
9114 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9115 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9116
9117 ins_cost(100); // XXX
9118 format %{ "setbn$cop $dst\t# signed, int" %}
9119 ins_encode %{
9120 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9121 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9122 %}
9123 ins_pipe(ialu_reg);
9124 %}
9125
9126 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
9127 %{
9128 predicate(!UseAPX);
9129 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9130
9131 ins_cost(200); // XXX
9132 format %{ "cmovl$cop $dst, $src\t# signed, int" %}
9133 ins_encode %{
9134 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9135 %}
9136 ins_pipe(pipe_cmov_reg);
9137 %}
9138
9139 instruct cmovI_reg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr, cmpOp cop)
9140 %{
9141 predicate(UseAPX);
9142 match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
9143
9144 ins_cost(200);
9145 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
9146 ins_encode %{
9147 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9148 %}
9149 ins_pipe(pipe_cmov_reg);
9150 %}
9151
9152 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
9153 %{
9154 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9155 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9156
9157 ins_cost(100); // XXX
9158 format %{ "setbn$cop $dst\t# unsigned, int" %}
9159 ins_encode %{
9160 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9161 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9162 %}
9163 ins_pipe(ialu_reg);
9164 %}
9165
9166 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
9167 predicate(!UseAPX);
9168 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9169
9170 ins_cost(200); // XXX
9171 format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
9172 ins_encode %{
9173 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9174 %}
9175 ins_pipe(pipe_cmov_reg);
9176 %}
9177
9178 instruct cmovI_regU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, rRegI src2) %{
9179 predicate(UseAPX);
9180 match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
9181
9182 ins_cost(200);
9183 format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
9184 ins_encode %{
9185 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9186 %}
9187 ins_pipe(pipe_cmov_reg);
9188 %}
9189
9190 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
9191 %{
9192 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9193 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9194
9195 ins_cost(100); // XXX
9196 format %{ "setbn$cop $dst\t# unsigned, int" %}
9197 ins_encode %{
9198 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9199 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9200 %}
9201 ins_pipe(ialu_reg);
9202 %}
9203
9204 instruct cmovI_imm_01UCFE(rRegI dst, immI_1 src, rFlagsRegUCFE cr, cmpOpUCFE cop)
9205 %{
9206 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9207 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9208
9209 ins_cost(100); // XXX
9210 format %{ "setbn$cop $dst\t# signed, unsigned, int" %}
9211 ins_encode %{
9212 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9213 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9214 %}
9215 ins_pipe(ialu_reg);
9216 %}
9217
9218 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
9219 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9220
9221 ins_cost(200);
9222 expand %{
9223 cmovI_regU(cop, cr, dst, src);
9224 %}
9225 %}
9226
9227 instruct cmovI_regUCFE_ndd(rRegI dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegI src1, rRegI src2) %{
9228 match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
9229
9230 ins_cost(200);
9231 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, unsigned, int ndd" %}
9232 ins_encode %{
9233 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9234 %}
9235 ins_pipe(pipe_cmov_reg);
9236 %}
9237
9238 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
9239 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9240 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9241
9242 ins_cost(200); // XXX
9243 format %{ "cmovpl $dst, $src\n\t"
9244 "cmovnel $dst, $src" %}
9245 ins_encode %{
9246 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9247 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9248 %}
9249 ins_pipe(pipe_cmov_reg);
9250 %}
9251
9252 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9253 // inputs of the CMove
9254 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
9255 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9256 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9257 effect(TEMP dst);
9258
9259 ins_cost(200); // XXX
9260 format %{ "cmovpl $dst, $src\n\t"
9261 "cmovnel $dst, $src" %}
9262 ins_encode %{
9263 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9264 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9265 %}
9266 ins_pipe(pipe_cmov_reg);
9267 %}
9268
9269 // Conditional move
9270 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
9271 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9272
9273 ins_cost(250); // XXX
9274 format %{ "cmovl$cop $dst, $src\t# signed, int" %}
9275 ins_encode %{
9276 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9277 %}
9278 ins_pipe(pipe_cmov_mem);
9279 %}
9280
9281 // Conditional move
9282 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
9283 %{
9284 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9285
9286 ins_cost(250); // XXX
9287 format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
9288 ins_encode %{
9289 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9290 %}
9291 ins_pipe(pipe_cmov_mem);
9292 %}
9293
9294 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
9295 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9296
9297 ins_cost(250);
9298 expand %{
9299 cmovI_memU(cop, cr, dst, src);
9300 %}
9301 %}
9302
9303 instruct cmovI_memUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, rRegI dst, memory src) %{
9304 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9305
9306 ins_cost(250); // XXX
9307 format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
9308 ins_encode %{
9309 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9310 %}
9311 ins_pipe(pipe_cmov_mem);
9312 %}
9313
9314 // Conditional move
9315 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
9316 %{
9317 predicate(!UseAPX);
9318 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9319
9320 ins_cost(200); // XXX
9321 format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
9322 ins_encode %{
9323 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9324 %}
9325 ins_pipe(pipe_cmov_reg);
9326 %}
9327
9328 // Conditional move ndd
9329 instruct cmovN_reg_ndd(rRegN dst, rRegN src1, rRegN src2, rFlagsReg cr, cmpOp cop)
9330 %{
9331 predicate(UseAPX);
9332 match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
9333
9334 ins_cost(200);
9335 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, compressed ptr ndd" %}
9336 ins_encode %{
9337 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9338 %}
9339 ins_pipe(pipe_cmov_reg);
9340 %}
9341
9342 // Conditional move
9343 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
9344 %{
9345 predicate(!UseAPX);
9346 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9347
9348 ins_cost(200); // XXX
9349 format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
9350 ins_encode %{
9351 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9352 %}
9353 ins_pipe(pipe_cmov_reg);
9354 %}
9355
9356 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
9357 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9358
9359 ins_cost(200);
9360 expand %{
9361 cmovN_regU(cop, cr, dst, src);
9362 %}
9363 %}
9364
9365 // Conditional move ndd
9366 instruct cmovN_regU_ndd(rRegN dst, cmpOpU cop, rFlagsRegU cr, rRegN src1, rRegN src2)
9367 %{
9368 predicate(UseAPX);
9369 match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
9370
9371 ins_cost(200);
9372 format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
9373 ins_encode %{
9374 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9375 %}
9376 ins_pipe(pipe_cmov_reg);
9377 %}
9378
9379 instruct cmovN_regUCFE_ndd(rRegN dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegN src1, rRegN src2) %{
9380 match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
9381
9382 ins_cost(200);
9383 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, unsigned, compressed ptr ndd" %}
9384 ins_encode %{
9385 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9386 %}
9387 ins_pipe(pipe_cmov_reg);
9388 %}
9389
9390 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
9391 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9392 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9393
9394 ins_cost(200); // XXX
9395 format %{ "cmovpl $dst, $src\n\t"
9396 "cmovnel $dst, $src" %}
9397 ins_encode %{
9398 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9399 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9400 %}
9401 ins_pipe(pipe_cmov_reg);
9402 %}
9403
9404 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9405 // inputs of the CMove
9406 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
9407 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9408 match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
9409
9410 ins_cost(200); // XXX
9411 format %{ "cmovpl $dst, $src\n\t"
9412 "cmovnel $dst, $src" %}
9413 ins_encode %{
9414 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9415 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9416 %}
9417 ins_pipe(pipe_cmov_reg);
9418 %}
9419
9420 // Conditional move
9421 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
9422 %{
9423 predicate(!UseAPX);
9424 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9425
9426 ins_cost(200); // XXX
9427 format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
9428 ins_encode %{
9429 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9430 %}
9431 ins_pipe(pipe_cmov_reg); // XXX
9432 %}
9433
9434 // Conditional move ndd
9435 instruct cmovP_reg_ndd(rRegP dst, rRegP src1, rRegP src2, rFlagsReg cr, cmpOp cop)
9436 %{
9437 predicate(UseAPX);
9438 match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
9439
9440 ins_cost(200);
9441 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, ptr ndd" %}
9442 ins_encode %{
9443 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9444 %}
9445 ins_pipe(pipe_cmov_reg);
9446 %}
9447
9448 // Conditional move
9449 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
9450 %{
9451 predicate(!UseAPX);
9452 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9453
9454 ins_cost(200); // XXX
9455 format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
9456 ins_encode %{
9457 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9458 %}
9459 ins_pipe(pipe_cmov_reg); // XXX
9460 %}
9461
9462 // Conditional move ndd
9463 instruct cmovP_regU_ndd(rRegP dst, cmpOpU cop, rFlagsRegU cr, rRegP src1, rRegP src2)
9464 %{
9465 predicate(UseAPX);
9466 match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
9467
9468 ins_cost(200);
9469 format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
9470 ins_encode %{
9471 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9472 %}
9473 ins_pipe(pipe_cmov_reg);
9474 %}
9475
9476 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
9477 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9478
9479 ins_cost(200);
9480 expand %{
9481 cmovP_regU(cop, cr, dst, src);
9482 %}
9483 %}
9484
9485 instruct cmovP_regUCFE_ndd(rRegP dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegP src1, rRegP src2) %{
9486 match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
9487
9488 ins_cost(200);
9489 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, unsigned, ptr ndd" %}
9490 ins_encode %{
9491 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9492 %}
9493 ins_pipe(pipe_cmov_reg);
9494 %}
9495
9496 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
9497 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9498 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9499
9500 ins_cost(200); // XXX
9501 format %{ "cmovpq $dst, $src\n\t"
9502 "cmovneq $dst, $src" %}
9503 ins_encode %{
9504 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9505 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9506 %}
9507 ins_pipe(pipe_cmov_reg);
9508 %}
9509
9510 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9511 // inputs of the CMove
9512 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
9513 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9514 match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
9515
9516 ins_cost(200); // XXX
9517 format %{ "cmovpq $dst, $src\n\t"
9518 "cmovneq $dst, $src" %}
9519 ins_encode %{
9520 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9521 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9522 %}
9523 ins_pipe(pipe_cmov_reg);
9524 %}
9525
9526 instruct cmovL_imm_01(rRegL dst, immL1 src, rFlagsReg cr, cmpOp cop)
9527 %{
9528 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9529 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9530
9531 ins_cost(100); // XXX
9532 format %{ "setbn$cop $dst\t# signed, long" %}
9533 ins_encode %{
9534 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9535 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9536 %}
9537 ins_pipe(ialu_reg);
9538 %}
9539
9540 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
9541 %{
9542 predicate(!UseAPX);
9543 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9544
9545 ins_cost(200); // XXX
9546 format %{ "cmovq$cop $dst, $src\t# signed, long" %}
9547 ins_encode %{
9548 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9549 %}
9550 ins_pipe(pipe_cmov_reg); // XXX
9551 %}
9552
9553 instruct cmovL_reg_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, rRegL src2)
9554 %{
9555 predicate(UseAPX);
9556 match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
9557
9558 ins_cost(200);
9559 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
9560 ins_encode %{
9561 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9562 %}
9563 ins_pipe(pipe_cmov_reg);
9564 %}
9565
9566 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
9567 %{
9568 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9569
9570 ins_cost(200); // XXX
9571 format %{ "cmovq$cop $dst, $src\t# signed, long" %}
9572 ins_encode %{
9573 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9574 %}
9575 ins_pipe(pipe_cmov_mem); // XXX
9576 %}
9577
9578 instruct cmovL_imm_01U(rRegL dst, immL1 src, rFlagsRegU cr, cmpOpU cop)
9579 %{
9580 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9581 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9582
9583 ins_cost(100); // XXX
9584 format %{ "setbn$cop $dst\t# unsigned, long" %}
9585 ins_encode %{
9586 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9587 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9588 %}
9589 ins_pipe(ialu_reg);
9590 %}
9591
9592 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
9593 %{
9594 predicate(!UseAPX);
9595 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9596
9597 ins_cost(200); // XXX
9598 format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
9599 ins_encode %{
9600 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9601 %}
9602 ins_pipe(pipe_cmov_reg); // XXX
9603 %}
9604
9605 instruct cmovL_regU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, rRegL src2)
9606 %{
9607 predicate(UseAPX);
9608 match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
9609
9610 ins_cost(200);
9611 format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
9612 ins_encode %{
9613 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9614 %}
9615 ins_pipe(pipe_cmov_reg);
9616 %}
9617
9618 instruct cmovL_imm_01UCF(rRegL dst, immL1 src, rFlagsRegUCF cr, cmpOpUCF cop)
9619 %{
9620 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9621 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9622
9623 ins_cost(100); // XXX
9624 format %{ "setbn$cop $dst\t# unsigned, long" %}
9625 ins_encode %{
9626 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9627 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9628 %}
9629 ins_pipe(ialu_reg);
9630 %}
9631
9632 instruct cmovL_imm_01UCFE(rRegL dst, immL1 src, rFlagsRegUCFE cr, cmpOpUCFE cop)
9633 %{
9634 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9635 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9636
9637 ins_cost(100); // XXX
9638 format %{ "setbn$cop $dst\t# signed, unsigned, long" %}
9639 ins_encode %{
9640 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9641 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9642 %}
9643 ins_pipe(ialu_reg);
9644 %}
9645
9646 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
9647 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9648
9649 ins_cost(200);
9650 expand %{
9651 cmovL_regU(cop, cr, dst, src);
9652 %}
9653 %}
9654
9655 instruct cmovL_regUCFE_ndd(rRegL dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegL src1, rRegL src2)
9656 %{
9657 match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
9658
9659 ins_cost(200);
9660 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, unsigned, long ndd" %}
9661 ins_encode %{
9662 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9663 %}
9664 ins_pipe(pipe_cmov_reg);
9665 %}
9666
9667 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
9668 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9669 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9670
9671 ins_cost(200); // XXX
9672 format %{ "cmovpq $dst, $src\n\t"
9673 "cmovneq $dst, $src" %}
9674 ins_encode %{
9675 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9676 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9677 %}
9678 ins_pipe(pipe_cmov_reg);
9679 %}
9680
9681 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9682 // inputs of the CMove
9683 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
9684 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9685 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9686
9687 ins_cost(200); // XXX
9688 format %{ "cmovpq $dst, $src\n\t"
9689 "cmovneq $dst, $src" %}
9690 ins_encode %{
9691 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9692 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9693 %}
9694 ins_pipe(pipe_cmov_reg);
9695 %}
9696
9697 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
9698 %{
9699 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9700
9701 ins_cost(200); // XXX
9702 format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
9703 ins_encode %{
9704 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9705 %}
9706 ins_pipe(pipe_cmov_mem); // XXX
9707 %}
9708
9709 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
9710 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9711
9712 ins_cost(200);
9713 expand %{
9714 cmovL_memU(cop, cr, dst, src);
9715 %}
9716 %}
9717
9718 instruct cmovL_memUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, rRegL dst, memory src) %{
9719 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9720
9721 ins_cost(200); // XXX
9722 format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
9723 ins_encode %{
9724 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9725 %}
9726 ins_pipe(pipe_cmov_mem); // XXX
9727 %}
9728
9729 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
9730 %{
9731 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9732
9733 ins_cost(200); // XXX
9734 format %{ "jn$cop skip\t# signed cmove float\n\t"
9735 "movss $dst, $src\n"
9736 "skip:" %}
9737 ins_encode %{
9738 Label Lskip;
9739 // Invert sense of branch from sense of CMOV
9740 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9741 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
9742 __ bind(Lskip);
9743 %}
9744 ins_pipe(pipe_slow);
9745 %}
9746
9747 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
9748 %{
9749 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9750
9751 ins_cost(200); // XXX
9752 format %{ "jn$cop skip\t# unsigned cmove float\n\t"
9753 "movss $dst, $src\n"
9754 "skip:" %}
9755 ins_encode %{
9756 Label Lskip;
9757 // Invert sense of branch from sense of CMOV
9758 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9759 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
9760 __ bind(Lskip);
9761 %}
9762 ins_pipe(pipe_slow);
9763 %}
9764
9765 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
9766 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9767
9768 ins_cost(200);
9769 expand %{
9770 cmovF_regU(cop, cr, dst, src);
9771 %}
9772 %}
9773
9774 instruct cmovF_regUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, regF dst, regF src)
9775 %{
9776 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9777
9778 ins_cost(200); // XXX
9779 format %{ "jn$cop skip\t# signed, unsigned cmove float\n\t"
9780 "movss $dst, $src\n"
9781 "skip:" %}
9782 ins_encode %{
9783 Label Lskip;
9784 // Invert sense of branch from sense of CMOV
9785 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9786 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
9787 __ bind(Lskip);
9788 %}
9789 ins_pipe(pipe_slow);
9790 %}
9791
9792 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
9793 %{
9794 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9795
9796 ins_cost(200); // XXX
9797 format %{ "jn$cop skip\t# signed cmove double\n\t"
9798 "movsd $dst, $src\n"
9799 "skip:" %}
9800 ins_encode %{
9801 Label Lskip;
9802 // Invert sense of branch from sense of CMOV
9803 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9804 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
9805 __ bind(Lskip);
9806 %}
9807 ins_pipe(pipe_slow);
9808 %}
9809
9810 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
9811 %{
9812 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9813
9814 ins_cost(200); // XXX
9815 format %{ "jn$cop skip\t# unsigned cmove double\n\t"
9816 "movsd $dst, $src\n"
9817 "skip:" %}
9818 ins_encode %{
9819 Label Lskip;
9820 // Invert sense of branch from sense of CMOV
9821 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9822 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
9823 __ bind(Lskip);
9824 %}
9825 ins_pipe(pipe_slow);
9826 %}
9827
9828 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
9829 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9830
9831 ins_cost(200);
9832 expand %{
9833 cmovD_regU(cop, cr, dst, src);
9834 %}
9835 %}
9836
9837 instruct cmovD_regUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, regD dst, regD src)
9838 %{
9839 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9840
9841 ins_cost(200); // XXX
9842 format %{ "jn$cop skip\t# signed, unsigned cmove double\n\t"
9843 "movsd $dst, $src\n"
9844 "skip:" %}
9845 ins_encode %{
9846 Label Lskip;
9847 // Invert sense of branch from sense of CMOV
9848 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9849 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
9850 __ bind(Lskip);
9851 %}
9852 ins_pipe(pipe_slow);
9853 %}
9854
9855 //----------Arithmetic Instructions--------------------------------------------
9856 //----------Addition Instructions----------------------------------------------
9857
9858 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9859 %{
9860 predicate(!UseAPX);
9861 match(Set dst (AddI dst src));
9862 effect(KILL cr);
9863 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9864 format %{ "addl $dst, $src\t# int" %}
9865 ins_encode %{
9866 __ addl($dst$$Register, $src$$Register);
9867 %}
9868 ins_pipe(ialu_reg_reg);
9869 %}
9870
9871 instruct addI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
9872 %{
9873 predicate(UseAPX);
9874 match(Set dst (AddI src1 src2));
9875 effect(KILL cr);
9876 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
9877
9878 format %{ "eaddl $dst, $src1, $src2\t# int ndd" %}
9879 ins_encode %{
9880 __ eaddl($dst$$Register, $src1$$Register, $src2$$Register, false);
9881 %}
9882 ins_pipe(ialu_reg_reg);
9883 %}
9884
9885 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9886 %{
9887 predicate(!UseAPX);
9888 match(Set dst (AddI dst src));
9889 effect(KILL cr);
9890 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9891
9892 format %{ "addl $dst, $src\t# int" %}
9893 ins_encode %{
9894 __ addl($dst$$Register, $src$$constant);
9895 %}
9896 ins_pipe( ialu_reg );
9897 %}
9898
9899 instruct addI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
9900 %{
9901 predicate(UseAPX);
9902 match(Set dst (AddI src1 src2));
9903 effect(KILL cr);
9904 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
9905
9906 format %{ "eaddl $dst, $src1, $src2\t# int ndd" %}
9907 ins_encode %{
9908 __ eaddl($dst$$Register, $src1$$Register, $src2$$constant, false);
9909 %}
9910 ins_pipe( ialu_reg );
9911 %}
9912
9913 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9914 %{
9915 match(Set dst (AddI dst (LoadI src)));
9916 effect(KILL cr);
9917 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9918
9919 ins_cost(150); // XXX
9920 format %{ "addl $dst, $src\t# int" %}
9921 ins_encode %{
9922 __ addl($dst$$Register, $src$$Address);
9923 %}
9924 ins_pipe(ialu_reg_mem);
9925 %}
9926
9927 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9928 %{
9929 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
9930 effect(KILL cr);
9931 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9932
9933 ins_cost(150); // XXX
9934 format %{ "addl $dst, $src\t# int" %}
9935 ins_encode %{
9936 __ addl($dst$$Address, $src$$Register);
9937 %}
9938 ins_pipe(ialu_mem_reg);
9939 %}
9940
9941 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
9942 %{
9943 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
9944 effect(KILL cr);
9945 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9946
9947
9948 ins_cost(125); // XXX
9949 format %{ "addl $dst, $src\t# int" %}
9950 ins_encode %{
9951 __ addl($dst$$Address, $src$$constant);
9952 %}
9953 ins_pipe(ialu_mem_imm);
9954 %}
9955
9956 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
9957 %{
9958 predicate(!UseAPX && UseIncDec);
9959 match(Set dst (AddI dst src));
9960 effect(KILL cr);
9961
9962 format %{ "incl $dst\t# int" %}
9963 ins_encode %{
9964 __ incrementl($dst$$Register);
9965 %}
9966 ins_pipe(ialu_reg);
9967 %}
9968
9969 instruct incI_rReg_ndd(rRegI dst, rRegI src, immI_1 val, rFlagsReg cr)
9970 %{
9971 predicate(UseAPX && UseIncDec);
9972 match(Set dst (AddI src val));
9973 effect(KILL cr);
9974 flag(PD::Flag_ndd_demotable_opr1);
9975
9976 format %{ "eincl $dst, $src\t# int ndd" %}
9977 ins_encode %{
9978 __ eincl($dst$$Register, $src$$Register, false);
9979 %}
9980 ins_pipe(ialu_reg);
9981 %}
9982
9983 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
9984 %{
9985 predicate(UseIncDec);
9986 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
9987 effect(KILL cr);
9988
9989 ins_cost(125); // XXX
9990 format %{ "incl $dst\t# int" %}
9991 ins_encode %{
9992 __ incrementl($dst$$Address);
9993 %}
9994 ins_pipe(ialu_mem_imm);
9995 %}
9996
9997 // XXX why does that use AddI
9998 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
9999 %{
10000 predicate(!UseAPX && UseIncDec);
10001 match(Set dst (AddI dst src));
10002 effect(KILL cr);
10003
10004 format %{ "decl $dst\t# int" %}
10005 ins_encode %{
10006 __ decrementl($dst$$Register);
10007 %}
10008 ins_pipe(ialu_reg);
10009 %}
10010
10011 instruct decI_rReg_ndd(rRegI dst, rRegI src, immI_M1 val, rFlagsReg cr)
10012 %{
10013 predicate(UseAPX && UseIncDec);
10014 match(Set dst (AddI src val));
10015 effect(KILL cr);
10016 flag(PD::Flag_ndd_demotable_opr1);
10017
10018 format %{ "edecl $dst, $src\t# int ndd" %}
10019 ins_encode %{
10020 __ edecl($dst$$Register, $src$$Register, false);
10021 %}
10022 ins_pipe(ialu_reg);
10023 %}
10024
10025 // XXX why does that use AddI
10026 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
10027 %{
10028 predicate(UseIncDec);
10029 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
10030 effect(KILL cr);
10031
10032 ins_cost(125); // XXX
10033 format %{ "decl $dst\t# int" %}
10034 ins_encode %{
10035 __ decrementl($dst$$Address);
10036 %}
10037 ins_pipe(ialu_mem_imm);
10038 %}
10039
10040 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
10041 %{
10042 predicate(VM_Version::supports_fast_2op_lea());
10043 match(Set dst (AddI (LShiftI index scale) disp));
10044
10045 format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
10046 ins_encode %{
10047 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10048 __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
10049 %}
10050 ins_pipe(ialu_reg_reg);
10051 %}
10052
10053 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
10054 %{
10055 predicate(VM_Version::supports_fast_3op_lea());
10056 match(Set dst (AddI (AddI base index) disp));
10057
10058 format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
10059 ins_encode %{
10060 __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
10061 %}
10062 ins_pipe(ialu_reg_reg);
10063 %}
10064
10065 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
10066 %{
10067 predicate(VM_Version::supports_fast_2op_lea());
10068 match(Set dst (AddI base (LShiftI index scale)));
10069
10070 format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
10071 ins_encode %{
10072 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10073 __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
10074 %}
10075 ins_pipe(ialu_reg_reg);
10076 %}
10077
10078 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
10079 %{
10080 predicate(VM_Version::supports_fast_3op_lea());
10081 match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
10082
10083 format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
10084 ins_encode %{
10085 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10086 __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
10087 %}
10088 ins_pipe(ialu_reg_reg);
10089 %}
10090
10091 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10092 %{
10093 predicate(!UseAPX);
10094 match(Set dst (AddL dst src));
10095 effect(KILL cr);
10096 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10097
10098 format %{ "addq $dst, $src\t# long" %}
10099 ins_encode %{
10100 __ addq($dst$$Register, $src$$Register);
10101 %}
10102 ins_pipe(ialu_reg_reg);
10103 %}
10104
10105 instruct addL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
10106 %{
10107 predicate(UseAPX);
10108 match(Set dst (AddL src1 src2));
10109 effect(KILL cr);
10110 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
10111
10112 format %{ "eaddq $dst, $src1, $src2\t# long ndd" %}
10113 ins_encode %{
10114 __ eaddq($dst$$Register, $src1$$Register, $src2$$Register, false);
10115 %}
10116 ins_pipe(ialu_reg_reg);
10117 %}
10118
10119 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10120 %{
10121 predicate(!UseAPX);
10122 match(Set dst (AddL dst src));
10123 effect(KILL cr);
10124 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10125
10126 format %{ "addq $dst, $src\t# long" %}
10127 ins_encode %{
10128 __ addq($dst$$Register, $src$$constant);
10129 %}
10130 ins_pipe( ialu_reg );
10131 %}
10132
10133 instruct addL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
10134 %{
10135 predicate(UseAPX);
10136 match(Set dst (AddL src1 src2));
10137 effect(KILL cr);
10138 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
10139
10140 format %{ "eaddq $dst, $src1, $src2\t# long ndd" %}
10141 ins_encode %{
10142 __ eaddq($dst$$Register, $src1$$Register, $src2$$constant, false);
10143 %}
10144 ins_pipe( ialu_reg );
10145 %}
10146
10147 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10148 %{
10149 match(Set dst (AddL dst (LoadL src)));
10150 effect(KILL cr);
10151 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10152
10153 ins_cost(150); // XXX
10154 format %{ "addq $dst, $src\t# long" %}
10155 ins_encode %{
10156 __ addq($dst$$Register, $src$$Address);
10157 %}
10158 ins_pipe(ialu_reg_mem);
10159 %}
10160
10161 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10162 %{
10163 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10164 effect(KILL cr);
10165 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10166
10167 ins_cost(150); // XXX
10168 format %{ "addq $dst, $src\t# long" %}
10169 ins_encode %{
10170 __ addq($dst$$Address, $src$$Register);
10171 %}
10172 ins_pipe(ialu_mem_reg);
10173 %}
10174
10175 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10176 %{
10177 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10178 effect(KILL cr);
10179 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10180
10181 ins_cost(125); // XXX
10182 format %{ "addq $dst, $src\t# long" %}
10183 ins_encode %{
10184 __ addq($dst$$Address, $src$$constant);
10185 %}
10186 ins_pipe(ialu_mem_imm);
10187 %}
10188
10189 instruct incL_rReg(rRegL dst, immL1 src, rFlagsReg cr)
10190 %{
10191 predicate(!UseAPX && UseIncDec);
10192 match(Set dst (AddL dst src));
10193 effect(KILL cr);
10194
10195 format %{ "incq $dst\t# long" %}
10196 ins_encode %{
10197 __ incrementq($dst$$Register);
10198 %}
10199 ins_pipe(ialu_reg);
10200 %}
10201
10202 instruct incL_rReg_ndd(rRegL dst, rRegI src, immL1 val, rFlagsReg cr)
10203 %{
10204 predicate(UseAPX && UseIncDec);
10205 match(Set dst (AddL src val));
10206 effect(KILL cr);
10207 flag(PD::Flag_ndd_demotable_opr1);
10208
10209 format %{ "eincq $dst, $src\t# long ndd" %}
10210 ins_encode %{
10211 __ eincq($dst$$Register, $src$$Register, false);
10212 %}
10213 ins_pipe(ialu_reg);
10214 %}
10215
10216 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
10217 %{
10218 predicate(UseIncDec);
10219 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10220 effect(KILL cr);
10221
10222 ins_cost(125); // XXX
10223 format %{ "incq $dst\t# long" %}
10224 ins_encode %{
10225 __ incrementq($dst$$Address);
10226 %}
10227 ins_pipe(ialu_mem_imm);
10228 %}
10229
10230 // XXX why does that use AddL
10231 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
10232 %{
10233 predicate(!UseAPX && UseIncDec);
10234 match(Set dst (AddL dst src));
10235 effect(KILL cr);
10236
10237 format %{ "decq $dst\t# long" %}
10238 ins_encode %{
10239 __ decrementq($dst$$Register);
10240 %}
10241 ins_pipe(ialu_reg);
10242 %}
10243
10244 instruct decL_rReg_ndd(rRegL dst, rRegL src, immL_M1 val, rFlagsReg cr)
10245 %{
10246 predicate(UseAPX && UseIncDec);
10247 match(Set dst (AddL src val));
10248 effect(KILL cr);
10249 flag(PD::Flag_ndd_demotable_opr1);
10250
10251 format %{ "edecq $dst, $src\t# long ndd" %}
10252 ins_encode %{
10253 __ edecq($dst$$Register, $src$$Register, false);
10254 %}
10255 ins_pipe(ialu_reg);
10256 %}
10257
10258 // XXX why does that use AddL
10259 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
10260 %{
10261 predicate(UseIncDec);
10262 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10263 effect(KILL cr);
10264
10265 ins_cost(125); // XXX
10266 format %{ "decq $dst\t# long" %}
10267 ins_encode %{
10268 __ decrementq($dst$$Address);
10269 %}
10270 ins_pipe(ialu_mem_imm);
10271 %}
10272
10273 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
10274 %{
10275 predicate(VM_Version::supports_fast_2op_lea());
10276 match(Set dst (AddL (LShiftL index scale) disp));
10277
10278 format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
10279 ins_encode %{
10280 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10281 __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
10282 %}
10283 ins_pipe(ialu_reg_reg);
10284 %}
10285
10286 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
10287 %{
10288 predicate(VM_Version::supports_fast_3op_lea());
10289 match(Set dst (AddL (AddL base index) disp));
10290
10291 format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
10292 ins_encode %{
10293 __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
10294 %}
10295 ins_pipe(ialu_reg_reg);
10296 %}
10297
10298 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
10299 %{
10300 predicate(VM_Version::supports_fast_2op_lea());
10301 match(Set dst (AddL base (LShiftL index scale)));
10302
10303 format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
10304 ins_encode %{
10305 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10306 __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
10307 %}
10308 ins_pipe(ialu_reg_reg);
10309 %}
10310
10311 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
10312 %{
10313 predicate(VM_Version::supports_fast_3op_lea());
10314 match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
10315
10316 format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
10317 ins_encode %{
10318 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10319 __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
10320 %}
10321 ins_pipe(ialu_reg_reg);
10322 %}
10323
10324 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
10325 %{
10326 match(Set dst (AddP dst src));
10327 effect(KILL cr);
10328 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10329
10330 format %{ "addq $dst, $src\t# ptr" %}
10331 ins_encode %{
10332 __ addq($dst$$Register, $src$$Register);
10333 %}
10334 ins_pipe(ialu_reg_reg);
10335 %}
10336
10337 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
10338 %{
10339 match(Set dst (AddP dst src));
10340 effect(KILL cr);
10341 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10342
10343 format %{ "addq $dst, $src\t# ptr" %}
10344 ins_encode %{
10345 __ addq($dst$$Register, $src$$constant);
10346 %}
10347 ins_pipe( ialu_reg );
10348 %}
10349
10350 // XXX addP mem ops ????
10351
10352 instruct checkCastPP(rRegP dst)
10353 %{
10354 match(Set dst (CheckCastPP dst));
10355
10356 size(0);
10357 format %{ "# checkcastPP of $dst" %}
10358 ins_encode(/* empty encoding */);
10359 ins_pipe(empty);
10360 %}
10361
10362 instruct castPP(rRegP dst)
10363 %{
10364 match(Set dst (CastPP dst));
10365
10366 size(0);
10367 format %{ "# castPP of $dst" %}
10368 ins_encode(/* empty encoding */);
10369 ins_pipe(empty);
10370 %}
10371
10372 instruct castII(rRegI dst)
10373 %{
10374 predicate(VerifyConstraintCasts == 0);
10375 match(Set dst (CastII dst));
10376
10377 size(0);
10378 format %{ "# castII of $dst" %}
10379 ins_encode(/* empty encoding */);
10380 ins_cost(0);
10381 ins_pipe(empty);
10382 %}
10383
10384 instruct castII_checked(rRegI dst, rFlagsReg cr)
10385 %{
10386 predicate(VerifyConstraintCasts > 0);
10387 match(Set dst (CastII dst));
10388
10389 effect(KILL cr);
10390 format %{ "# cast_checked_II $dst" %}
10391 ins_encode %{
10392 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register);
10393 %}
10394 ins_pipe(pipe_slow);
10395 %}
10396
10397 instruct castLL(rRegL dst)
10398 %{
10399 predicate(VerifyConstraintCasts == 0);
10400 match(Set dst (CastLL dst));
10401
10402 size(0);
10403 format %{ "# castLL of $dst" %}
10404 ins_encode(/* empty encoding */);
10405 ins_cost(0);
10406 ins_pipe(empty);
10407 %}
10408
10409 instruct castLL_checked_L32(rRegL dst, rFlagsReg cr)
10410 %{
10411 predicate(VerifyConstraintCasts > 0 && castLL_is_imm32(n));
10412 match(Set dst (CastLL dst));
10413
10414 effect(KILL cr);
10415 format %{ "# cast_checked_LL $dst" %}
10416 ins_encode %{
10417 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, noreg);
10418 %}
10419 ins_pipe(pipe_slow);
10420 %}
10421
10422 instruct castLL_checked(rRegL dst, rRegL tmp, rFlagsReg cr)
10423 %{
10424 predicate(VerifyConstraintCasts > 0 && !castLL_is_imm32(n));
10425 match(Set dst (CastLL dst));
10426
10427 effect(KILL cr, TEMP tmp);
10428 format %{ "# cast_checked_LL $dst\tusing $tmp as TEMP" %}
10429 ins_encode %{
10430 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, $tmp$$Register);
10431 %}
10432 ins_pipe(pipe_slow);
10433 %}
10434
10435 instruct castFF(regF dst)
10436 %{
10437 match(Set dst (CastFF dst));
10438
10439 size(0);
10440 format %{ "# castFF of $dst" %}
10441 ins_encode(/* empty encoding */);
10442 ins_cost(0);
10443 ins_pipe(empty);
10444 %}
10445
10446 instruct castHH(regF dst)
10447 %{
10448 match(Set dst (CastHH dst));
10449
10450 size(0);
10451 format %{ "# castHH of $dst" %}
10452 ins_encode(/* empty encoding */);
10453 ins_cost(0);
10454 ins_pipe(empty);
10455 %}
10456
10457 instruct castDD(regD dst)
10458 %{
10459 match(Set dst (CastDD dst));
10460
10461 size(0);
10462 format %{ "# castDD of $dst" %}
10463 ins_encode(/* empty encoding */);
10464 ins_cost(0);
10465 ins_pipe(empty);
10466 %}
10467
10468 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
10469 instruct compareAndSwapP(rRegI res,
10470 memory mem_ptr,
10471 rax_RegP oldval, rRegP newval,
10472 rFlagsReg cr)
10473 %{
10474 predicate(n->as_LoadStore()->barrier_data() == 0);
10475 match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
10476 match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
10477 effect(KILL cr, KILL oldval);
10478
10479 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10480 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10481 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10482 ins_encode %{
10483 __ lock();
10484 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10485 __ setcc(Assembler::equal, $res$$Register);
10486 %}
10487 ins_pipe( pipe_cmpxchg );
10488 %}
10489
10490 instruct compareAndSwapL(rRegI res,
10491 memory mem_ptr,
10492 rax_RegL oldval, rRegL newval,
10493 rFlagsReg cr)
10494 %{
10495 match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
10496 match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
10497 effect(KILL cr, KILL oldval);
10498
10499 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10500 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10501 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10502 ins_encode %{
10503 __ lock();
10504 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10505 __ setcc(Assembler::equal, $res$$Register);
10506 %}
10507 ins_pipe( pipe_cmpxchg );
10508 %}
10509
10510 instruct compareAndSwapI(rRegI res,
10511 memory mem_ptr,
10512 rax_RegI oldval, rRegI newval,
10513 rFlagsReg cr)
10514 %{
10515 match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
10516 match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
10517 effect(KILL cr, KILL oldval);
10518
10519 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10520 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10521 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10522 ins_encode %{
10523 __ lock();
10524 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10525 __ setcc(Assembler::equal, $res$$Register);
10526 %}
10527 ins_pipe( pipe_cmpxchg );
10528 %}
10529
10530 instruct compareAndSwapB(rRegI res,
10531 memory mem_ptr,
10532 rax_RegI oldval, rRegI newval,
10533 rFlagsReg cr)
10534 %{
10535 match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
10536 match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
10537 effect(KILL cr, KILL oldval);
10538
10539 format %{ "cmpxchgb $mem_ptr,$newval\t# "
10540 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10541 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10542 ins_encode %{
10543 __ lock();
10544 __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
10545 __ setcc(Assembler::equal, $res$$Register);
10546 %}
10547 ins_pipe( pipe_cmpxchg );
10548 %}
10549
10550 instruct compareAndSwapS(rRegI res,
10551 memory mem_ptr,
10552 rax_RegI oldval, rRegI newval,
10553 rFlagsReg cr)
10554 %{
10555 match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
10556 match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
10557 effect(KILL cr, KILL oldval);
10558
10559 format %{ "cmpxchgw $mem_ptr,$newval\t# "
10560 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10561 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10562 ins_encode %{
10563 __ lock();
10564 __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
10565 __ setcc(Assembler::equal, $res$$Register);
10566 %}
10567 ins_pipe( pipe_cmpxchg );
10568 %}
10569
10570 instruct compareAndSwapN(rRegI res,
10571 memory mem_ptr,
10572 rax_RegN oldval, rRegN newval,
10573 rFlagsReg cr) %{
10574 predicate(n->as_LoadStore()->barrier_data() == 0);
10575 match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
10576 match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
10577 effect(KILL cr, KILL oldval);
10578
10579 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10580 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10581 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10582 ins_encode %{
10583 __ lock();
10584 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10585 __ setcc(Assembler::equal, $res$$Register);
10586 %}
10587 ins_pipe( pipe_cmpxchg );
10588 %}
10589
10590 instruct compareAndExchangeB(
10591 memory mem_ptr,
10592 rax_RegI oldval, rRegI newval,
10593 rFlagsReg cr)
10594 %{
10595 match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
10596 effect(KILL cr);
10597
10598 format %{ "cmpxchgb $mem_ptr,$newval\t# "
10599 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10600 ins_encode %{
10601 __ lock();
10602 __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
10603 %}
10604 ins_pipe( pipe_cmpxchg );
10605 %}
10606
10607 instruct compareAndExchangeS(
10608 memory mem_ptr,
10609 rax_RegI oldval, rRegI newval,
10610 rFlagsReg cr)
10611 %{
10612 match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
10613 effect(KILL cr);
10614
10615 format %{ "cmpxchgw $mem_ptr,$newval\t# "
10616 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10617 ins_encode %{
10618 __ lock();
10619 __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
10620 %}
10621 ins_pipe( pipe_cmpxchg );
10622 %}
10623
10624 instruct compareAndExchangeI(
10625 memory mem_ptr,
10626 rax_RegI oldval, rRegI newval,
10627 rFlagsReg cr)
10628 %{
10629 match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
10630 effect(KILL cr);
10631
10632 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10633 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10634 ins_encode %{
10635 __ lock();
10636 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10637 %}
10638 ins_pipe( pipe_cmpxchg );
10639 %}
10640
10641 instruct compareAndExchangeL(
10642 memory mem_ptr,
10643 rax_RegL oldval, rRegL newval,
10644 rFlagsReg cr)
10645 %{
10646 match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
10647 effect(KILL cr);
10648
10649 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10650 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10651 ins_encode %{
10652 __ lock();
10653 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10654 %}
10655 ins_pipe( pipe_cmpxchg );
10656 %}
10657
10658 instruct compareAndExchangeN(
10659 memory mem_ptr,
10660 rax_RegN oldval, rRegN newval,
10661 rFlagsReg cr) %{
10662 predicate(n->as_LoadStore()->barrier_data() == 0);
10663 match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
10664 effect(KILL cr);
10665
10666 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10667 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10668 ins_encode %{
10669 __ lock();
10670 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10671 %}
10672 ins_pipe( pipe_cmpxchg );
10673 %}
10674
10675 instruct compareAndExchangeP(
10676 memory mem_ptr,
10677 rax_RegP oldval, rRegP newval,
10678 rFlagsReg cr)
10679 %{
10680 predicate(n->as_LoadStore()->barrier_data() == 0);
10681 match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
10682 effect(KILL cr);
10683
10684 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10685 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10686 ins_encode %{
10687 __ lock();
10688 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10689 %}
10690 ins_pipe( pipe_cmpxchg );
10691 %}
10692
10693 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
10694 predicate(n->as_LoadStore()->result_not_used());
10695 match(Set dummy (GetAndAddB mem add));
10696 effect(KILL cr);
10697 format %{ "addb_lock $mem, $add" %}
10698 ins_encode %{
10699 __ lock();
10700 __ addb($mem$$Address, $add$$Register);
10701 %}
10702 ins_pipe(pipe_cmpxchg);
10703 %}
10704
10705 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
10706 predicate(n->as_LoadStore()->result_not_used());
10707 match(Set dummy (GetAndAddB mem add));
10708 effect(KILL cr);
10709 format %{ "addb_lock $mem, $add" %}
10710 ins_encode %{
10711 __ lock();
10712 __ addb($mem$$Address, $add$$constant);
10713 %}
10714 ins_pipe(pipe_cmpxchg);
10715 %}
10716
10717 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
10718 predicate(!n->as_LoadStore()->result_not_used());
10719 match(Set newval (GetAndAddB mem newval));
10720 effect(KILL cr);
10721 format %{ "xaddb_lock $mem, $newval\t# $newval -> byte" %}
10722 ins_encode %{
10723 __ lock();
10724 __ xaddb($mem$$Address, $newval$$Register);
10725 __ narrow_subword_type($newval$$Register, T_BYTE);
10726 %}
10727 ins_pipe(pipe_cmpxchg);
10728 %}
10729
10730 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
10731 predicate(n->as_LoadStore()->result_not_used());
10732 match(Set dummy (GetAndAddS mem add));
10733 effect(KILL cr);
10734 format %{ "addw_lock $mem, $add" %}
10735 ins_encode %{
10736 __ lock();
10737 __ addw($mem$$Address, $add$$Register);
10738 %}
10739 ins_pipe(pipe_cmpxchg);
10740 %}
10741
10742 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
10743 predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
10744 match(Set dummy (GetAndAddS mem add));
10745 effect(KILL cr);
10746 format %{ "addw_lock $mem, $add" %}
10747 ins_encode %{
10748 __ lock();
10749 __ addw($mem$$Address, $add$$constant);
10750 %}
10751 ins_pipe(pipe_cmpxchg);
10752 %}
10753
10754 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
10755 predicate(!n->as_LoadStore()->result_not_used());
10756 match(Set newval (GetAndAddS mem newval));
10757 effect(KILL cr);
10758 format %{ "xaddw_lock $mem, $newval\t# $newval -> short" %}
10759 ins_encode %{
10760 __ lock();
10761 __ xaddw($mem$$Address, $newval$$Register);
10762 __ narrow_subword_type($newval$$Register, T_SHORT);
10763 %}
10764 ins_pipe(pipe_cmpxchg);
10765 %}
10766
10767 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
10768 predicate(n->as_LoadStore()->result_not_used());
10769 match(Set dummy (GetAndAddI mem add));
10770 effect(KILL cr);
10771 format %{ "addl_lock $mem, $add" %}
10772 ins_encode %{
10773 __ lock();
10774 __ addl($mem$$Address, $add$$Register);
10775 %}
10776 ins_pipe(pipe_cmpxchg);
10777 %}
10778
10779 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
10780 predicate(n->as_LoadStore()->result_not_used());
10781 match(Set dummy (GetAndAddI mem add));
10782 effect(KILL cr);
10783 format %{ "addl_lock $mem, $add" %}
10784 ins_encode %{
10785 __ lock();
10786 __ addl($mem$$Address, $add$$constant);
10787 %}
10788 ins_pipe(pipe_cmpxchg);
10789 %}
10790
10791 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
10792 predicate(!n->as_LoadStore()->result_not_used());
10793 match(Set newval (GetAndAddI mem newval));
10794 effect(KILL cr);
10795 format %{ "xaddl_lock $mem, $newval" %}
10796 ins_encode %{
10797 __ lock();
10798 __ xaddl($mem$$Address, $newval$$Register);
10799 %}
10800 ins_pipe(pipe_cmpxchg);
10801 %}
10802
10803 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
10804 predicate(n->as_LoadStore()->result_not_used());
10805 match(Set dummy (GetAndAddL mem add));
10806 effect(KILL cr);
10807 format %{ "addq_lock $mem, $add" %}
10808 ins_encode %{
10809 __ lock();
10810 __ addq($mem$$Address, $add$$Register);
10811 %}
10812 ins_pipe(pipe_cmpxchg);
10813 %}
10814
10815 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
10816 predicate(n->as_LoadStore()->result_not_used());
10817 match(Set dummy (GetAndAddL mem add));
10818 effect(KILL cr);
10819 format %{ "addq_lock $mem, $add" %}
10820 ins_encode %{
10821 __ lock();
10822 __ addq($mem$$Address, $add$$constant);
10823 %}
10824 ins_pipe(pipe_cmpxchg);
10825 %}
10826
10827 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
10828 predicate(!n->as_LoadStore()->result_not_used());
10829 match(Set newval (GetAndAddL mem newval));
10830 effect(KILL cr);
10831 format %{ "xaddq_lock $mem, $newval" %}
10832 ins_encode %{
10833 __ lock();
10834 __ xaddq($mem$$Address, $newval$$Register);
10835 %}
10836 ins_pipe(pipe_cmpxchg);
10837 %}
10838
10839 instruct xchgB( memory mem, rRegI newval) %{
10840 match(Set newval (GetAndSetB mem newval));
10841 format %{ "XCHGB $newval,[$mem]\t# $newval -> byte" %}
10842 ins_encode %{
10843 __ xchgb($newval$$Register, $mem$$Address);
10844 __ narrow_subword_type($newval$$Register, T_BYTE);
10845 %}
10846 ins_pipe( pipe_cmpxchg );
10847 %}
10848
10849 instruct xchgS( memory mem, rRegI newval) %{
10850 match(Set newval (GetAndSetS mem newval));
10851 format %{ "XCHGW $newval,[$mem]\t# $newval -> short" %}
10852 ins_encode %{
10853 __ xchgw($newval$$Register, $mem$$Address);
10854 __ narrow_subword_type($newval$$Register, T_SHORT);
10855 %}
10856 ins_pipe( pipe_cmpxchg );
10857 %}
10858
10859 instruct xchgI( memory mem, rRegI newval) %{
10860 match(Set newval (GetAndSetI mem newval));
10861 format %{ "XCHGL $newval,[$mem]" %}
10862 ins_encode %{
10863 __ xchgl($newval$$Register, $mem$$Address);
10864 %}
10865 ins_pipe( pipe_cmpxchg );
10866 %}
10867
10868 instruct xchgL( memory mem, rRegL newval) %{
10869 match(Set newval (GetAndSetL mem newval));
10870 format %{ "XCHGL $newval,[$mem]" %}
10871 ins_encode %{
10872 __ xchgq($newval$$Register, $mem$$Address);
10873 %}
10874 ins_pipe( pipe_cmpxchg );
10875 %}
10876
10877 instruct xchgP( memory mem, rRegP newval) %{
10878 match(Set newval (GetAndSetP mem newval));
10879 predicate(n->as_LoadStore()->barrier_data() == 0);
10880 format %{ "XCHGQ $newval,[$mem]" %}
10881 ins_encode %{
10882 __ xchgq($newval$$Register, $mem$$Address);
10883 %}
10884 ins_pipe( pipe_cmpxchg );
10885 %}
10886
10887 instruct xchgN( memory mem, rRegN newval) %{
10888 predicate(n->as_LoadStore()->barrier_data() == 0);
10889 match(Set newval (GetAndSetN mem newval));
10890 format %{ "XCHGL $newval,$mem]" %}
10891 ins_encode %{
10892 __ xchgl($newval$$Register, $mem$$Address);
10893 %}
10894 ins_pipe( pipe_cmpxchg );
10895 %}
10896
10897 //----------Abs Instructions-------------------------------------------
10898
10899 // Integer Absolute Instructions
10900 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10901 %{
10902 match(Set dst (AbsI src));
10903 effect(TEMP dst, KILL cr);
10904 format %{ "xorl $dst, $dst\t# abs int\n\t"
10905 "subl $dst, $src\n\t"
10906 "cmovll $dst, $src" %}
10907 ins_encode %{
10908 __ xorl($dst$$Register, $dst$$Register);
10909 __ subl($dst$$Register, $src$$Register);
10910 __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
10911 %}
10912
10913 ins_pipe(ialu_reg_reg);
10914 %}
10915
10916 // Long Absolute Instructions
10917 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10918 %{
10919 match(Set dst (AbsL src));
10920 effect(TEMP dst, KILL cr);
10921 format %{ "xorl $dst, $dst\t# abs long\n\t"
10922 "subq $dst, $src\n\t"
10923 "cmovlq $dst, $src" %}
10924 ins_encode %{
10925 __ xorl($dst$$Register, $dst$$Register);
10926 __ subq($dst$$Register, $src$$Register);
10927 __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
10928 %}
10929
10930 ins_pipe(ialu_reg_reg);
10931 %}
10932
10933 //----------Subtraction Instructions-------------------------------------------
10934
10935 // Integer Subtraction Instructions
10936 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10937 %{
10938 predicate(!UseAPX);
10939 match(Set dst (SubI dst src));
10940 effect(KILL cr);
10941 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10942
10943 format %{ "subl $dst, $src\t# int" %}
10944 ins_encode %{
10945 __ subl($dst$$Register, $src$$Register);
10946 %}
10947 ins_pipe(ialu_reg_reg);
10948 %}
10949
10950 instruct subI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10951 %{
10952 predicate(UseAPX);
10953 match(Set dst (SubI src1 src2));
10954 effect(KILL cr);
10955 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
10956
10957 format %{ "esubl $dst, $src1, $src2\t# int ndd" %}
10958 ins_encode %{
10959 __ esubl($dst$$Register, $src1$$Register, $src2$$Register, false);
10960 %}
10961 ins_pipe(ialu_reg_reg);
10962 %}
10963
10964 instruct subI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10965 %{
10966 predicate(UseAPX);
10967 match(Set dst (SubI src1 src2));
10968 effect(KILL cr);
10969 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
10970
10971 format %{ "esubl $dst, $src1, $src2\t# int ndd" %}
10972 ins_encode %{
10973 __ esubl($dst$$Register, $src1$$Register, $src2$$constant, false);
10974 %}
10975 ins_pipe(ialu_reg_reg);
10976 %}
10977
10978 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10979 %{
10980 match(Set dst (SubI dst (LoadI src)));
10981 effect(KILL cr);
10982 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10983
10984 ins_cost(150);
10985 format %{ "subl $dst, $src\t# int" %}
10986 ins_encode %{
10987 __ subl($dst$$Register, $src$$Address);
10988 %}
10989 ins_pipe(ialu_reg_mem);
10990 %}
10991
10992 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10993 %{
10994 match(Set dst (StoreI dst (SubI (LoadI dst) src)));
10995 effect(KILL cr);
10996 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10997
10998 ins_cost(150);
10999 format %{ "subl $dst, $src\t# int" %}
11000 ins_encode %{
11001 __ subl($dst$$Address, $src$$Register);
11002 %}
11003 ins_pipe(ialu_mem_reg);
11004 %}
11005
11006 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11007 %{
11008 predicate(!UseAPX);
11009 match(Set dst (SubL dst src));
11010 effect(KILL cr);
11011 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11012
11013 format %{ "subq $dst, $src\t# long" %}
11014 ins_encode %{
11015 __ subq($dst$$Register, $src$$Register);
11016 %}
11017 ins_pipe(ialu_reg_reg);
11018 %}
11019
11020 instruct subL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11021 %{
11022 predicate(UseAPX);
11023 match(Set dst (SubL src1 src2));
11024 effect(KILL cr);
11025 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11026
11027 format %{ "esubq $dst, $src1, $src2\t# long ndd" %}
11028 ins_encode %{
11029 __ esubq($dst$$Register, $src1$$Register, $src2$$Register, false);
11030 %}
11031 ins_pipe(ialu_reg_reg);
11032 %}
11033
11034 instruct subL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11035 %{
11036 predicate(UseAPX);
11037 match(Set dst (SubL src1 src2));
11038 effect(KILL cr);
11039 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11040
11041 format %{ "esubq $dst, $src1, $src2\t# long ndd" %}
11042 ins_encode %{
11043 __ esubq($dst$$Register, $src1$$Register, $src2$$constant, false);
11044 %}
11045 ins_pipe(ialu_reg_reg);
11046 %}
11047
11048 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11049 %{
11050 match(Set dst (SubL dst (LoadL src)));
11051 effect(KILL cr);
11052 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11053
11054 ins_cost(150);
11055 format %{ "subq $dst, $src\t# long" %}
11056 ins_encode %{
11057 __ subq($dst$$Register, $src$$Address);
11058 %}
11059 ins_pipe(ialu_reg_mem);
11060 %}
11061
11062 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11063 %{
11064 match(Set dst (StoreL dst (SubL (LoadL dst) src)));
11065 effect(KILL cr);
11066 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11067
11068 ins_cost(150);
11069 format %{ "subq $dst, $src\t# long" %}
11070 ins_encode %{
11071 __ subq($dst$$Address, $src$$Register);
11072 %}
11073 ins_pipe(ialu_mem_reg);
11074 %}
11075
11076 // Subtract from a pointer
11077 // XXX hmpf???
11078 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
11079 %{
11080 match(Set dst (AddP dst (SubI zero src)));
11081 effect(KILL cr);
11082
11083 format %{ "subq $dst, $src\t# ptr - int" %}
11084 ins_encode %{
11085 __ subq($dst$$Register, $src$$Register);
11086 %}
11087 ins_pipe(ialu_reg_reg);
11088 %}
11089
11090 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
11091 %{
11092 predicate(!UseAPX);
11093 match(Set dst (SubI zero dst));
11094 effect(KILL cr);
11095 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11096
11097 format %{ "negl $dst\t# int" %}
11098 ins_encode %{
11099 __ negl($dst$$Register);
11100 %}
11101 ins_pipe(ialu_reg);
11102 %}
11103
11104 instruct negI_rReg_ndd(rRegI dst, rRegI src, immI_0 zero, rFlagsReg cr)
11105 %{
11106 predicate(UseAPX);
11107 match(Set dst (SubI zero src));
11108 effect(KILL cr);
11109 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr2);
11110
11111 format %{ "enegl $dst, $src\t# int ndd" %}
11112 ins_encode %{
11113 __ enegl($dst$$Register, $src$$Register, false);
11114 %}
11115 ins_pipe(ialu_reg);
11116 %}
11117
11118 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
11119 %{
11120 predicate(!UseAPX);
11121 match(Set dst (NegI dst));
11122 effect(KILL cr);
11123 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11124
11125 format %{ "negl $dst\t# int" %}
11126 ins_encode %{
11127 __ negl($dst$$Register);
11128 %}
11129 ins_pipe(ialu_reg);
11130 %}
11131
11132 instruct negI_rReg_2_ndd(rRegI dst, rRegI src, rFlagsReg cr)
11133 %{
11134 predicate(UseAPX);
11135 match(Set dst (NegI src));
11136 effect(KILL cr);
11137 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11138
11139 format %{ "enegl $dst, $src\t# int ndd" %}
11140 ins_encode %{
11141 __ enegl($dst$$Register, $src$$Register, false);
11142 %}
11143 ins_pipe(ialu_reg);
11144 %}
11145
11146 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
11147 %{
11148 match(Set dst (StoreI dst (SubI zero (LoadI dst))));
11149 effect(KILL cr);
11150 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11151
11152 format %{ "negl $dst\t# int" %}
11153 ins_encode %{
11154 __ negl($dst$$Address);
11155 %}
11156 ins_pipe(ialu_reg);
11157 %}
11158
11159 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
11160 %{
11161 predicate(!UseAPX);
11162 match(Set dst (SubL zero dst));
11163 effect(KILL cr);
11164 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11165
11166 format %{ "negq $dst\t# long" %}
11167 ins_encode %{
11168 __ negq($dst$$Register);
11169 %}
11170 ins_pipe(ialu_reg);
11171 %}
11172
11173 instruct negL_rReg_ndd(rRegL dst, rRegL src, immL0 zero, rFlagsReg cr)
11174 %{
11175 predicate(UseAPX);
11176 match(Set dst (SubL zero src));
11177 effect(KILL cr);
11178 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr2);
11179
11180 format %{ "enegq $dst, $src\t# long ndd" %}
11181 ins_encode %{
11182 __ enegq($dst$$Register, $src$$Register, false);
11183 %}
11184 ins_pipe(ialu_reg);
11185 %}
11186
11187 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
11188 %{
11189 predicate(!UseAPX);
11190 match(Set dst (NegL dst));
11191 effect(KILL cr);
11192 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11193
11194 format %{ "negq $dst\t# int" %}
11195 ins_encode %{
11196 __ negq($dst$$Register);
11197 %}
11198 ins_pipe(ialu_reg);
11199 %}
11200
11201 instruct negL_rReg_2_ndd(rRegL dst, rRegL src, rFlagsReg cr)
11202 %{
11203 predicate(UseAPX);
11204 match(Set dst (NegL src));
11205 effect(KILL cr);
11206 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11207
11208 format %{ "enegq $dst, $src\t# long ndd" %}
11209 ins_encode %{
11210 __ enegq($dst$$Register, $src$$Register, false);
11211 %}
11212 ins_pipe(ialu_reg);
11213 %}
11214
11215 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
11216 %{
11217 match(Set dst (StoreL dst (SubL zero (LoadL dst))));
11218 effect(KILL cr);
11219 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11220
11221 format %{ "negq $dst\t# long" %}
11222 ins_encode %{
11223 __ negq($dst$$Address);
11224 %}
11225 ins_pipe(ialu_reg);
11226 %}
11227
11228 //----------Multiplication/Division Instructions-------------------------------
11229 // Integer Multiplication Instructions
11230 // Multiply Register
11231
11232 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
11233 %{
11234 predicate(!UseAPX);
11235 match(Set dst (MulI dst src));
11236 effect(KILL cr);
11237
11238 ins_cost(300);
11239 format %{ "imull $dst, $src\t# int" %}
11240 ins_encode %{
11241 __ imull($dst$$Register, $src$$Register);
11242 %}
11243 ins_pipe(ialu_reg_reg_alu0);
11244 %}
11245
11246 instruct mulI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
11247 %{
11248 predicate(UseAPX);
11249 match(Set dst (MulI src1 src2));
11250 effect(KILL cr);
11251 flag(PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
11252
11253 ins_cost(300);
11254 format %{ "eimull $dst, $src1, $src2\t# int ndd" %}
11255 ins_encode %{
11256 __ eimull($dst$$Register, $src1$$Register, $src2$$Register, false);
11257 %}
11258 ins_pipe(ialu_reg_reg_alu0);
11259 %}
11260
11261 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
11262 %{
11263 match(Set dst (MulI src imm));
11264 effect(KILL cr);
11265
11266 ins_cost(300);
11267 format %{ "imull $dst, $src, $imm\t# int" %}
11268 ins_encode %{
11269 __ imull($dst$$Register, $src$$Register, $imm$$constant);
11270 %}
11271 ins_pipe(ialu_reg_reg_alu0);
11272 %}
11273
11274 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
11275 %{
11276 match(Set dst (MulI dst (LoadI src)));
11277 effect(KILL cr);
11278
11279 ins_cost(350);
11280 format %{ "imull $dst, $src\t# int" %}
11281 ins_encode %{
11282 __ imull($dst$$Register, $src$$Address);
11283 %}
11284 ins_pipe(ialu_reg_mem_alu0);
11285 %}
11286
11287 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
11288 %{
11289 match(Set dst (MulI (LoadI src) imm));
11290 effect(KILL cr);
11291
11292 ins_cost(300);
11293 format %{ "imull $dst, $src, $imm\t# int" %}
11294 ins_encode %{
11295 __ imull($dst$$Register, $src$$Address, $imm$$constant);
11296 %}
11297 ins_pipe(ialu_reg_mem_alu0);
11298 %}
11299
11300 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
11301 %{
11302 match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
11303 effect(KILL cr, KILL src2);
11304
11305 expand %{ mulI_rReg(dst, src1, cr);
11306 mulI_rReg(src2, src3, cr);
11307 addI_rReg(dst, src2, cr); %}
11308 %}
11309
11310 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11311 %{
11312 predicate(!UseAPX);
11313 match(Set dst (MulL dst src));
11314 effect(KILL cr);
11315
11316 ins_cost(300);
11317 format %{ "imulq $dst, $src\t# long" %}
11318 ins_encode %{
11319 __ imulq($dst$$Register, $src$$Register);
11320 %}
11321 ins_pipe(ialu_reg_reg_alu0);
11322 %}
11323
11324 instruct mulL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11325 %{
11326 predicate(UseAPX);
11327 match(Set dst (MulL src1 src2));
11328 effect(KILL cr);
11329 flag(PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
11330
11331 ins_cost(300);
11332 format %{ "eimulq $dst, $src1, $src2\t# long ndd" %}
11333 ins_encode %{
11334 __ eimulq($dst$$Register, $src1$$Register, $src2$$Register, false);
11335 %}
11336 ins_pipe(ialu_reg_reg_alu0);
11337 %}
11338
11339 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
11340 %{
11341 match(Set dst (MulL src imm));
11342 effect(KILL cr);
11343
11344 ins_cost(300);
11345 format %{ "imulq $dst, $src, $imm\t# long" %}
11346 ins_encode %{
11347 __ imulq($dst$$Register, $src$$Register, $imm$$constant);
11348 %}
11349 ins_pipe(ialu_reg_reg_alu0);
11350 %}
11351
11352 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
11353 %{
11354 match(Set dst (MulL dst (LoadL src)));
11355 effect(KILL cr);
11356
11357 ins_cost(350);
11358 format %{ "imulq $dst, $src\t# long" %}
11359 ins_encode %{
11360 __ imulq($dst$$Register, $src$$Address);
11361 %}
11362 ins_pipe(ialu_reg_mem_alu0);
11363 %}
11364
11365
11366 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
11367 %{
11368 match(Set dst (MulL (LoadL src) imm));
11369 effect(KILL cr);
11370
11371 ins_cost(300);
11372 format %{ "imulq $dst, $src, $imm\t# long" %}
11373 ins_encode %{
11374 __ imulq($dst$$Register, $src$$Address, $imm$$constant);
11375 %}
11376 ins_pipe(ialu_reg_mem_alu0);
11377 %}
11378
11379 instruct mulHiLoL_rReg(rax_RegL rax, rdx_RegL rdx, rRegL src, rFlagsReg cr)
11380 %{
11381 match(MulHiLoL src rax);
11382 match(MulHiLoL rax src);
11383 effect(KILL cr);
11384
11385 ins_cost(300);
11386 format %{ "imulq RDX:RAX, RAX, $src\t# mulhilo" %}
11387 ins_encode %{
11388 __ imulq($src$$Register);
11389 %}
11390 ins_pipe(ialu_reg_reg_alu0);
11391 %}
11392
11393 instruct umulHiLoL_rReg(rax_RegL rax, rdx_RegL rdx, rRegL src, rFlagsReg cr)
11394 %{
11395 match(UMulHiLoL src rax);
11396 match(UMulHiLoL rax src);
11397 effect(KILL cr);
11398
11399 ins_cost(300);
11400 format %{ "mulq RDX:RAX, RAX, $src\t# umulhilo" %}
11401 ins_encode %{
11402 __ mulq($src$$Register);
11403 %}
11404 ins_pipe(ialu_reg_reg_alu0);
11405 %}
11406
11407 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
11408 %{
11409 match(Set dst (MulHiL src rax));
11410 effect(USE_KILL rax, KILL cr);
11411
11412 ins_cost(300);
11413 format %{ "imulq RDX:RAX, RAX, $src\t# mulhi" %}
11414 ins_encode %{
11415 __ imulq($src$$Register);
11416 %}
11417 ins_pipe(ialu_reg_reg_alu0);
11418 %}
11419
11420 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
11421 %{
11422 match(Set dst (UMulHiL src rax));
11423 effect(USE_KILL rax, KILL cr);
11424
11425 ins_cost(300);
11426 format %{ "mulq RDX:RAX, RAX, $src\t# umulhi" %}
11427 ins_encode %{
11428 __ mulq($src$$Register);
11429 %}
11430 ins_pipe(ialu_reg_reg_alu0);
11431 %}
11432
11433 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
11434 rFlagsReg cr)
11435 %{
11436 match(Set rax (DivI rax div));
11437 effect(KILL rdx, KILL cr);
11438
11439 ins_cost(30*100+10*100); // XXX
11440 format %{ "cmpl rax, 0x80000000\t# idiv\n\t"
11441 "jne,s normal\n\t"
11442 "xorl rdx, rdx\n\t"
11443 "cmpl $div, -1\n\t"
11444 "je,s done\n"
11445 "normal: cdql\n\t"
11446 "idivl $div\n"
11447 "done:" %}
11448 ins_encode(cdql_enc(div));
11449 ins_pipe(ialu_reg_reg_alu0);
11450 %}
11451
11452 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
11453 rFlagsReg cr)
11454 %{
11455 match(Set rax (DivL rax div));
11456 effect(KILL rdx, KILL cr);
11457
11458 ins_cost(30*100+10*100); // XXX
11459 format %{ "movq rdx, 0x8000000000000000\t# ldiv\n\t"
11460 "cmpq rax, rdx\n\t"
11461 "jne,s normal\n\t"
11462 "xorl rdx, rdx\n\t"
11463 "cmpq $div, -1\n\t"
11464 "je,s done\n"
11465 "normal: cdqq\n\t"
11466 "idivq $div\n"
11467 "done:" %}
11468 ins_encode(cdqq_enc(div));
11469 ins_pipe(ialu_reg_reg_alu0);
11470 %}
11471
11472 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
11473 %{
11474 match(Set rax (UDivI rax div));
11475 effect(KILL rdx, KILL cr);
11476
11477 ins_cost(300);
11478 format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
11479 ins_encode %{
11480 __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
11481 %}
11482 ins_pipe(ialu_reg_reg_alu0);
11483 %}
11484
11485 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
11486 %{
11487 match(Set rax (UDivL rax div));
11488 effect(KILL rdx, KILL cr);
11489
11490 ins_cost(300);
11491 format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
11492 ins_encode %{
11493 __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
11494 %}
11495 ins_pipe(ialu_reg_reg_alu0);
11496 %}
11497
11498 // Integer DIVMOD with Register, both quotient and mod results
11499 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
11500 rFlagsReg cr)
11501 %{
11502 match(DivModI rax div);
11503 effect(KILL cr);
11504
11505 ins_cost(30*100+10*100); // XXX
11506 format %{ "cmpl rax, 0x80000000\t# idiv\n\t"
11507 "jne,s normal\n\t"
11508 "xorl rdx, rdx\n\t"
11509 "cmpl $div, -1\n\t"
11510 "je,s done\n"
11511 "normal: cdql\n\t"
11512 "idivl $div\n"
11513 "done:" %}
11514 ins_encode(cdql_enc(div));
11515 ins_pipe(pipe_slow);
11516 %}
11517
11518 // Long DIVMOD with Register, both quotient and mod results
11519 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
11520 rFlagsReg cr)
11521 %{
11522 match(DivModL rax div);
11523 effect(KILL cr);
11524
11525 ins_cost(30*100+10*100); // XXX
11526 format %{ "movq rdx, 0x8000000000000000\t# ldiv\n\t"
11527 "cmpq rax, rdx\n\t"
11528 "jne,s normal\n\t"
11529 "xorl rdx, rdx\n\t"
11530 "cmpq $div, -1\n\t"
11531 "je,s done\n"
11532 "normal: cdqq\n\t"
11533 "idivq $div\n"
11534 "done:" %}
11535 ins_encode(cdqq_enc(div));
11536 ins_pipe(pipe_slow);
11537 %}
11538
11539 // Unsigned integer DIVMOD with Register, both quotient and mod results
11540 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
11541 no_rax_rdx_RegI div, rFlagsReg cr)
11542 %{
11543 match(UDivModI rax div);
11544 effect(TEMP tmp, KILL cr);
11545
11546 ins_cost(300);
11547 format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
11548 "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
11549 %}
11550 ins_encode %{
11551 __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
11552 %}
11553 ins_pipe(pipe_slow);
11554 %}
11555
11556 // Unsigned long DIVMOD with Register, both quotient and mod results
11557 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
11558 no_rax_rdx_RegL div, rFlagsReg cr)
11559 %{
11560 match(UDivModL rax div);
11561 effect(TEMP tmp, KILL cr);
11562
11563 ins_cost(300);
11564 format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
11565 "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
11566 %}
11567 ins_encode %{
11568 __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
11569 %}
11570 ins_pipe(pipe_slow);
11571 %}
11572
11573 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
11574 rFlagsReg cr)
11575 %{
11576 match(Set rdx (ModI rax div));
11577 effect(KILL rax, KILL cr);
11578
11579 ins_cost(300); // XXX
11580 format %{ "cmpl rax, 0x80000000\t# irem\n\t"
11581 "jne,s normal\n\t"
11582 "xorl rdx, rdx\n\t"
11583 "cmpl $div, -1\n\t"
11584 "je,s done\n"
11585 "normal: cdql\n\t"
11586 "idivl $div\n"
11587 "done:" %}
11588 ins_encode(cdql_enc(div));
11589 ins_pipe(ialu_reg_reg_alu0);
11590 %}
11591
11592 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
11593 rFlagsReg cr)
11594 %{
11595 match(Set rdx (ModL rax div));
11596 effect(KILL rax, KILL cr);
11597
11598 ins_cost(300); // XXX
11599 format %{ "movq rdx, 0x8000000000000000\t# lrem\n\t"
11600 "cmpq rax, rdx\n\t"
11601 "jne,s normal\n\t"
11602 "xorl rdx, rdx\n\t"
11603 "cmpq $div, -1\n\t"
11604 "je,s done\n"
11605 "normal: cdqq\n\t"
11606 "idivq $div\n"
11607 "done:" %}
11608 ins_encode(cdqq_enc(div));
11609 ins_pipe(ialu_reg_reg_alu0);
11610 %}
11611
11612 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
11613 %{
11614 match(Set rdx (UModI rax div));
11615 effect(KILL rax, KILL cr);
11616
11617 ins_cost(300);
11618 format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
11619 ins_encode %{
11620 __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
11621 %}
11622 ins_pipe(ialu_reg_reg_alu0);
11623 %}
11624
11625 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
11626 %{
11627 match(Set rdx (UModL rax div));
11628 effect(KILL rax, KILL cr);
11629
11630 ins_cost(300);
11631 format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
11632 ins_encode %{
11633 __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
11634 %}
11635 ins_pipe(ialu_reg_reg_alu0);
11636 %}
11637
11638 // Integer Shift Instructions
11639 // Shift Left by one, two, three
11640 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
11641 %{
11642 predicate(!UseAPX);
11643 match(Set dst (LShiftI dst shift));
11644 effect(KILL cr);
11645
11646 format %{ "sall $dst, $shift" %}
11647 ins_encode %{
11648 __ sall($dst$$Register, $shift$$constant);
11649 %}
11650 ins_pipe(ialu_reg);
11651 %}
11652
11653 // Shift Left by one, two, three
11654 instruct salI_rReg_immI2_ndd(rRegI dst, rRegI src, immI2 shift, rFlagsReg cr)
11655 %{
11656 predicate(UseAPX);
11657 match(Set dst (LShiftI src shift));
11658 effect(KILL cr);
11659 flag(PD::Flag_ndd_demotable_opr1);
11660
11661 format %{ "esall $dst, $src, $shift\t# int(ndd)" %}
11662 ins_encode %{
11663 __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
11664 %}
11665 ins_pipe(ialu_reg);
11666 %}
11667
11668 // Shift Left by 8-bit immediate
11669 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
11670 %{
11671 predicate(!UseAPX);
11672 match(Set dst (LShiftI dst shift));
11673 effect(KILL cr);
11674
11675 format %{ "sall $dst, $shift" %}
11676 ins_encode %{
11677 __ sall($dst$$Register, $shift$$constant);
11678 %}
11679 ins_pipe(ialu_reg);
11680 %}
11681
11682 // Shift Left by 8-bit immediate
11683 instruct salI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
11684 %{
11685 predicate(UseAPX);
11686 match(Set dst (LShiftI src shift));
11687 effect(KILL cr);
11688 flag(PD::Flag_ndd_demotable_opr1);
11689
11690 format %{ "esall $dst, $src, $shift\t# int (ndd)" %}
11691 ins_encode %{
11692 __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
11693 %}
11694 ins_pipe(ialu_reg);
11695 %}
11696
11697 // Shift Left by 8-bit immediate
11698 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
11699 %{
11700 match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
11701 effect(KILL cr);
11702
11703 format %{ "sall $dst, $shift" %}
11704 ins_encode %{
11705 __ sall($dst$$Address, $shift$$constant);
11706 %}
11707 ins_pipe(ialu_mem_imm);
11708 %}
11709
11710 // Shift Left by variable
11711 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
11712 %{
11713 predicate(!VM_Version::supports_bmi2());
11714 match(Set dst (LShiftI dst shift));
11715 effect(KILL cr);
11716
11717 format %{ "sall $dst, $shift" %}
11718 ins_encode %{
11719 __ sall($dst$$Register);
11720 %}
11721 ins_pipe(ialu_reg_reg);
11722 %}
11723
11724 // Shift Left by variable
11725 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
11726 %{
11727 predicate(!VM_Version::supports_bmi2());
11728 match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
11729 effect(KILL cr);
11730
11731 format %{ "sall $dst, $shift" %}
11732 ins_encode %{
11733 __ sall($dst$$Address);
11734 %}
11735 ins_pipe(ialu_mem_reg);
11736 %}
11737
11738 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
11739 %{
11740 predicate(VM_Version::supports_bmi2());
11741 match(Set dst (LShiftI src shift));
11742
11743 format %{ "shlxl $dst, $src, $shift" %}
11744 ins_encode %{
11745 __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
11746 %}
11747 ins_pipe(ialu_reg_reg);
11748 %}
11749
11750 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
11751 %{
11752 predicate(VM_Version::supports_bmi2());
11753 match(Set dst (LShiftI (LoadI src) shift));
11754 ins_cost(175);
11755 format %{ "shlxl $dst, $src, $shift" %}
11756 ins_encode %{
11757 __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
11758 %}
11759 ins_pipe(ialu_reg_mem);
11760 %}
11761
11762 // Arithmetic Shift Right by 8-bit immediate
11763 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
11764 %{
11765 predicate(!UseAPX);
11766 match(Set dst (RShiftI dst shift));
11767 effect(KILL cr);
11768
11769 format %{ "sarl $dst, $shift" %}
11770 ins_encode %{
11771 __ sarl($dst$$Register, $shift$$constant);
11772 %}
11773 ins_pipe(ialu_mem_imm);
11774 %}
11775
11776 // Arithmetic Shift Right by 8-bit immediate
11777 instruct sarI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
11778 %{
11779 predicate(UseAPX);
11780 match(Set dst (RShiftI src shift));
11781 effect(KILL cr);
11782 flag(PD::Flag_ndd_demotable_opr1);
11783
11784 format %{ "esarl $dst, $src, $shift\t# int (ndd)" %}
11785 ins_encode %{
11786 __ esarl($dst$$Register, $src$$Register, $shift$$constant, false);
11787 %}
11788 ins_pipe(ialu_mem_imm);
11789 %}
11790
11791 // Arithmetic Shift Right by 8-bit immediate
11792 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
11793 %{
11794 match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
11795 effect(KILL cr);
11796
11797 format %{ "sarl $dst, $shift" %}
11798 ins_encode %{
11799 __ sarl($dst$$Address, $shift$$constant);
11800 %}
11801 ins_pipe(ialu_mem_imm);
11802 %}
11803
11804 // Arithmetic Shift Right by variable
11805 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
11806 %{
11807 predicate(!VM_Version::supports_bmi2());
11808 match(Set dst (RShiftI dst shift));
11809 effect(KILL cr);
11810
11811 format %{ "sarl $dst, $shift" %}
11812 ins_encode %{
11813 __ sarl($dst$$Register);
11814 %}
11815 ins_pipe(ialu_reg_reg);
11816 %}
11817
11818 // Arithmetic Shift Right by variable
11819 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
11820 %{
11821 predicate(!VM_Version::supports_bmi2());
11822 match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
11823 effect(KILL cr);
11824
11825 format %{ "sarl $dst, $shift" %}
11826 ins_encode %{
11827 __ sarl($dst$$Address);
11828 %}
11829 ins_pipe(ialu_mem_reg);
11830 %}
11831
11832 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
11833 %{
11834 predicate(VM_Version::supports_bmi2());
11835 match(Set dst (RShiftI src shift));
11836
11837 format %{ "sarxl $dst, $src, $shift" %}
11838 ins_encode %{
11839 __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
11840 %}
11841 ins_pipe(ialu_reg_reg);
11842 %}
11843
11844 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
11845 %{
11846 predicate(VM_Version::supports_bmi2());
11847 match(Set dst (RShiftI (LoadI src) shift));
11848 ins_cost(175);
11849 format %{ "sarxl $dst, $src, $shift" %}
11850 ins_encode %{
11851 __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
11852 %}
11853 ins_pipe(ialu_reg_mem);
11854 %}
11855
11856 // Logical Shift Right by 8-bit immediate
11857 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
11858 %{
11859 predicate(!UseAPX);
11860 match(Set dst (URShiftI dst shift));
11861 effect(KILL cr);
11862
11863 format %{ "shrl $dst, $shift" %}
11864 ins_encode %{
11865 __ shrl($dst$$Register, $shift$$constant);
11866 %}
11867 ins_pipe(ialu_reg);
11868 %}
11869
11870 // Logical Shift Right by 8-bit immediate
11871 instruct shrI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
11872 %{
11873 predicate(UseAPX);
11874 match(Set dst (URShiftI src shift));
11875 effect(KILL cr);
11876 flag(PD::Flag_ndd_demotable_opr1);
11877
11878 format %{ "eshrl $dst, $src, $shift\t # int (ndd)" %}
11879 ins_encode %{
11880 __ eshrl($dst$$Register, $src$$Register, $shift$$constant, false);
11881 %}
11882 ins_pipe(ialu_reg);
11883 %}
11884
11885 // Logical Shift Right by 8-bit immediate
11886 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
11887 %{
11888 match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
11889 effect(KILL cr);
11890
11891 format %{ "shrl $dst, $shift" %}
11892 ins_encode %{
11893 __ shrl($dst$$Address, $shift$$constant);
11894 %}
11895 ins_pipe(ialu_mem_imm);
11896 %}
11897
11898 // Logical Shift Right by variable
11899 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
11900 %{
11901 predicate(!VM_Version::supports_bmi2());
11902 match(Set dst (URShiftI dst shift));
11903 effect(KILL cr);
11904
11905 format %{ "shrl $dst, $shift" %}
11906 ins_encode %{
11907 __ shrl($dst$$Register);
11908 %}
11909 ins_pipe(ialu_reg_reg);
11910 %}
11911
11912 // Logical Shift Right by variable
11913 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
11914 %{
11915 predicate(!VM_Version::supports_bmi2());
11916 match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
11917 effect(KILL cr);
11918
11919 format %{ "shrl $dst, $shift" %}
11920 ins_encode %{
11921 __ shrl($dst$$Address);
11922 %}
11923 ins_pipe(ialu_mem_reg);
11924 %}
11925
11926 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
11927 %{
11928 predicate(VM_Version::supports_bmi2());
11929 match(Set dst (URShiftI src shift));
11930
11931 format %{ "shrxl $dst, $src, $shift" %}
11932 ins_encode %{
11933 __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
11934 %}
11935 ins_pipe(ialu_reg_reg);
11936 %}
11937
11938 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
11939 %{
11940 predicate(VM_Version::supports_bmi2());
11941 match(Set dst (URShiftI (LoadI src) shift));
11942 ins_cost(175);
11943 format %{ "shrxl $dst, $src, $shift" %}
11944 ins_encode %{
11945 __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
11946 %}
11947 ins_pipe(ialu_reg_mem);
11948 %}
11949
11950 // Long Shift Instructions
11951 // Shift Left by one, two, three
11952 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
11953 %{
11954 predicate(!UseAPX);
11955 match(Set dst (LShiftL dst shift));
11956 effect(KILL cr);
11957
11958 format %{ "salq $dst, $shift" %}
11959 ins_encode %{
11960 __ salq($dst$$Register, $shift$$constant);
11961 %}
11962 ins_pipe(ialu_reg);
11963 %}
11964
11965 // Shift Left by one, two, three
11966 instruct salL_rReg_immI2_ndd(rRegL dst, rRegL src, immI2 shift, rFlagsReg cr)
11967 %{
11968 predicate(UseAPX);
11969 match(Set dst (LShiftL src shift));
11970 effect(KILL cr);
11971 flag(PD::Flag_ndd_demotable_opr1);
11972
11973 format %{ "esalq $dst, $src, $shift\t# long (ndd)" %}
11974 ins_encode %{
11975 __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
11976 %}
11977 ins_pipe(ialu_reg);
11978 %}
11979
11980 // Shift Left by 8-bit immediate
11981 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
11982 %{
11983 predicate(!UseAPX);
11984 match(Set dst (LShiftL dst shift));
11985 effect(KILL cr);
11986
11987 format %{ "salq $dst, $shift" %}
11988 ins_encode %{
11989 __ salq($dst$$Register, $shift$$constant);
11990 %}
11991 ins_pipe(ialu_reg);
11992 %}
11993
11994 // Shift Left by 8-bit immediate
11995 instruct salL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
11996 %{
11997 predicate(UseAPX);
11998 match(Set dst (LShiftL src shift));
11999 effect(KILL cr);
12000 flag(PD::Flag_ndd_demotable_opr1);
12001
12002 format %{ "esalq $dst, $src, $shift\t# long (ndd)" %}
12003 ins_encode %{
12004 __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
12005 %}
12006 ins_pipe(ialu_reg);
12007 %}
12008
12009 // Shift Left by 8-bit immediate
12010 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
12011 %{
12012 match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
12013 effect(KILL cr);
12014
12015 format %{ "salq $dst, $shift" %}
12016 ins_encode %{
12017 __ salq($dst$$Address, $shift$$constant);
12018 %}
12019 ins_pipe(ialu_mem_imm);
12020 %}
12021
12022 // Shift Left by variable
12023 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12024 %{
12025 predicate(!VM_Version::supports_bmi2());
12026 match(Set dst (LShiftL dst shift));
12027 effect(KILL cr);
12028
12029 format %{ "salq $dst, $shift" %}
12030 ins_encode %{
12031 __ salq($dst$$Register);
12032 %}
12033 ins_pipe(ialu_reg_reg);
12034 %}
12035
12036 // Shift Left by variable
12037 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
12038 %{
12039 predicate(!VM_Version::supports_bmi2());
12040 match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
12041 effect(KILL cr);
12042
12043 format %{ "salq $dst, $shift" %}
12044 ins_encode %{
12045 __ salq($dst$$Address);
12046 %}
12047 ins_pipe(ialu_mem_reg);
12048 %}
12049
12050 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
12051 %{
12052 predicate(VM_Version::supports_bmi2());
12053 match(Set dst (LShiftL src shift));
12054
12055 format %{ "shlxq $dst, $src, $shift" %}
12056 ins_encode %{
12057 __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
12058 %}
12059 ins_pipe(ialu_reg_reg);
12060 %}
12061
12062 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
12063 %{
12064 predicate(VM_Version::supports_bmi2());
12065 match(Set dst (LShiftL (LoadL src) shift));
12066 ins_cost(175);
12067 format %{ "shlxq $dst, $src, $shift" %}
12068 ins_encode %{
12069 __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
12070 %}
12071 ins_pipe(ialu_reg_mem);
12072 %}
12073
12074 // Arithmetic Shift Right by 8-bit immediate
12075 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
12076 %{
12077 predicate(!UseAPX);
12078 match(Set dst (RShiftL dst shift));
12079 effect(KILL cr);
12080
12081 format %{ "sarq $dst, $shift" %}
12082 ins_encode %{
12083 __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
12084 %}
12085 ins_pipe(ialu_mem_imm);
12086 %}
12087
12088 // Arithmetic Shift Right by 8-bit immediate
12089 instruct sarL_rReg_imm_ndd(rRegL dst, rRegL src, immI shift, rFlagsReg cr)
12090 %{
12091 predicate(UseAPX);
12092 match(Set dst (RShiftL src shift));
12093 effect(KILL cr);
12094 flag(PD::Flag_ndd_demotable_opr1);
12095
12096 format %{ "esarq $dst, $src, $shift\t# long (ndd)" %}
12097 ins_encode %{
12098 __ esarq($dst$$Register, $src$$Register, (unsigned char)($shift$$constant & 0x3F), false);
12099 %}
12100 ins_pipe(ialu_mem_imm);
12101 %}
12102
12103 // Arithmetic Shift Right by 8-bit immediate
12104 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
12105 %{
12106 match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
12107 effect(KILL cr);
12108
12109 format %{ "sarq $dst, $shift" %}
12110 ins_encode %{
12111 __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
12112 %}
12113 ins_pipe(ialu_mem_imm);
12114 %}
12115
12116 // Arithmetic Shift Right by variable
12117 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12118 %{
12119 predicate(!VM_Version::supports_bmi2());
12120 match(Set dst (RShiftL dst shift));
12121 effect(KILL cr);
12122
12123 format %{ "sarq $dst, $shift" %}
12124 ins_encode %{
12125 __ sarq($dst$$Register);
12126 %}
12127 ins_pipe(ialu_reg_reg);
12128 %}
12129
12130 // Arithmetic Shift Right by variable
12131 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
12132 %{
12133 predicate(!VM_Version::supports_bmi2());
12134 match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
12135 effect(KILL cr);
12136
12137 format %{ "sarq $dst, $shift" %}
12138 ins_encode %{
12139 __ sarq($dst$$Address);
12140 %}
12141 ins_pipe(ialu_mem_reg);
12142 %}
12143
12144 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
12145 %{
12146 predicate(VM_Version::supports_bmi2());
12147 match(Set dst (RShiftL src shift));
12148
12149 format %{ "sarxq $dst, $src, $shift" %}
12150 ins_encode %{
12151 __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
12152 %}
12153 ins_pipe(ialu_reg_reg);
12154 %}
12155
12156 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
12157 %{
12158 predicate(VM_Version::supports_bmi2());
12159 match(Set dst (RShiftL (LoadL src) shift));
12160 ins_cost(175);
12161 format %{ "sarxq $dst, $src, $shift" %}
12162 ins_encode %{
12163 __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
12164 %}
12165 ins_pipe(ialu_reg_mem);
12166 %}
12167
12168 // Logical Shift Right by 8-bit immediate
12169 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
12170 %{
12171 predicate(!UseAPX);
12172 match(Set dst (URShiftL dst shift));
12173 effect(KILL cr);
12174
12175 format %{ "shrq $dst, $shift" %}
12176 ins_encode %{
12177 __ shrq($dst$$Register, $shift$$constant);
12178 %}
12179 ins_pipe(ialu_reg);
12180 %}
12181
12182 // Logical Shift Right by 8-bit immediate
12183 instruct shrL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
12184 %{
12185 predicate(UseAPX);
12186 match(Set dst (URShiftL src shift));
12187 effect(KILL cr);
12188 flag(PD::Flag_ndd_demotable_opr1);
12189
12190 format %{ "eshrq $dst, $src, $shift\t# long (ndd)" %}
12191 ins_encode %{
12192 __ eshrq($dst$$Register, $src$$Register, $shift$$constant, false);
12193 %}
12194 ins_pipe(ialu_reg);
12195 %}
12196
12197 // Logical Shift Right by 8-bit immediate
12198 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
12199 %{
12200 match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
12201 effect(KILL cr);
12202
12203 format %{ "shrq $dst, $shift" %}
12204 ins_encode %{
12205 __ shrq($dst$$Address, $shift$$constant);
12206 %}
12207 ins_pipe(ialu_mem_imm);
12208 %}
12209
12210 // Logical Shift Right by variable
12211 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12212 %{
12213 predicate(!VM_Version::supports_bmi2());
12214 match(Set dst (URShiftL dst shift));
12215 effect(KILL cr);
12216
12217 format %{ "shrq $dst, $shift" %}
12218 ins_encode %{
12219 __ shrq($dst$$Register);
12220 %}
12221 ins_pipe(ialu_reg_reg);
12222 %}
12223
12224 // Logical Shift Right by variable
12225 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
12226 %{
12227 predicate(!VM_Version::supports_bmi2());
12228 match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
12229 effect(KILL cr);
12230
12231 format %{ "shrq $dst, $shift" %}
12232 ins_encode %{
12233 __ shrq($dst$$Address);
12234 %}
12235 ins_pipe(ialu_mem_reg);
12236 %}
12237
12238 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
12239 %{
12240 predicate(VM_Version::supports_bmi2());
12241 match(Set dst (URShiftL src shift));
12242
12243 format %{ "shrxq $dst, $src, $shift" %}
12244 ins_encode %{
12245 __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
12246 %}
12247 ins_pipe(ialu_reg_reg);
12248 %}
12249
12250 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
12251 %{
12252 predicate(VM_Version::supports_bmi2());
12253 match(Set dst (URShiftL (LoadL src) shift));
12254 ins_cost(175);
12255 format %{ "shrxq $dst, $src, $shift" %}
12256 ins_encode %{
12257 __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
12258 %}
12259 ins_pipe(ialu_reg_mem);
12260 %}
12261
12262 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
12263 // This idiom is used by the compiler for the i2b bytecode.
12264 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
12265 %{
12266 match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
12267
12268 format %{ "movsbl $dst, $src\t# i2b" %}
12269 ins_encode %{
12270 __ movsbl($dst$$Register, $src$$Register);
12271 %}
12272 ins_pipe(ialu_reg_reg);
12273 %}
12274
12275 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
12276 // This idiom is used by the compiler the i2s bytecode.
12277 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
12278 %{
12279 match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
12280
12281 format %{ "movswl $dst, $src\t# i2s" %}
12282 ins_encode %{
12283 __ movswl($dst$$Register, $src$$Register);
12284 %}
12285 ins_pipe(ialu_reg_reg);
12286 %}
12287
12288 // ROL/ROR instructions
12289
12290 // Rotate left by constant.
12291 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
12292 %{
12293 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12294 match(Set dst (RotateLeft dst shift));
12295 effect(KILL cr);
12296 format %{ "roll $dst, $shift" %}
12297 ins_encode %{
12298 __ roll($dst$$Register, $shift$$constant);
12299 %}
12300 ins_pipe(ialu_reg);
12301 %}
12302
12303 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
12304 %{
12305 predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12306 match(Set dst (RotateLeft src shift));
12307 format %{ "rolxl $dst, $src, $shift" %}
12308 ins_encode %{
12309 int shift = 32 - ($shift$$constant & 31);
12310 __ rorxl($dst$$Register, $src$$Register, shift);
12311 %}
12312 ins_pipe(ialu_reg_reg);
12313 %}
12314
12315 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
12316 %{
12317 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12318 match(Set dst (RotateLeft (LoadI src) shift));
12319 ins_cost(175);
12320 format %{ "rolxl $dst, $src, $shift" %}
12321 ins_encode %{
12322 int shift = 32 - ($shift$$constant & 31);
12323 __ rorxl($dst$$Register, $src$$Address, shift);
12324 %}
12325 ins_pipe(ialu_reg_mem);
12326 %}
12327
12328 // Rotate Left by variable
12329 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
12330 %{
12331 predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
12332 match(Set dst (RotateLeft dst shift));
12333 effect(KILL cr);
12334 format %{ "roll $dst, $shift" %}
12335 ins_encode %{
12336 __ roll($dst$$Register);
12337 %}
12338 ins_pipe(ialu_reg_reg);
12339 %}
12340
12341 // Rotate Left by variable
12342 instruct rolI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
12343 %{
12344 predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
12345 match(Set dst (RotateLeft src shift));
12346 effect(KILL cr);
12347 flag(PD::Flag_ndd_demotable_opr1);
12348
12349 format %{ "eroll $dst, $src, $shift\t# rotate left (int ndd)" %}
12350 ins_encode %{
12351 __ eroll($dst$$Register, $src$$Register, false);
12352 %}
12353 ins_pipe(ialu_reg_reg);
12354 %}
12355
12356 // Rotate Right by constant.
12357 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
12358 %{
12359 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12360 match(Set dst (RotateRight dst shift));
12361 effect(KILL cr);
12362 format %{ "rorl $dst, $shift" %}
12363 ins_encode %{
12364 __ rorl($dst$$Register, $shift$$constant);
12365 %}
12366 ins_pipe(ialu_reg);
12367 %}
12368
12369 // Rotate Right by constant.
12370 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
12371 %{
12372 predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12373 match(Set dst (RotateRight src shift));
12374 format %{ "rorxl $dst, $src, $shift" %}
12375 ins_encode %{
12376 __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
12377 %}
12378 ins_pipe(ialu_reg_reg);
12379 %}
12380
12381 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
12382 %{
12383 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12384 match(Set dst (RotateRight (LoadI src) shift));
12385 ins_cost(175);
12386 format %{ "rorxl $dst, $src, $shift" %}
12387 ins_encode %{
12388 __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
12389 %}
12390 ins_pipe(ialu_reg_mem);
12391 %}
12392
12393 // Rotate Right by variable
12394 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
12395 %{
12396 predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
12397 match(Set dst (RotateRight dst shift));
12398 effect(KILL cr);
12399 format %{ "rorl $dst, $shift" %}
12400 ins_encode %{
12401 __ rorl($dst$$Register);
12402 %}
12403 ins_pipe(ialu_reg_reg);
12404 %}
12405
12406 // Rotate Right by variable
12407 instruct rorI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
12408 %{
12409 predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
12410 match(Set dst (RotateRight src shift));
12411 effect(KILL cr);
12412 flag(PD::Flag_ndd_demotable_opr1);
12413
12414 format %{ "erorl $dst, $src, $shift\t# rotate right(int ndd)" %}
12415 ins_encode %{
12416 __ erorl($dst$$Register, $src$$Register, false);
12417 %}
12418 ins_pipe(ialu_reg_reg);
12419 %}
12420
12421 // Rotate Left by constant.
12422 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
12423 %{
12424 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12425 match(Set dst (RotateLeft dst shift));
12426 effect(KILL cr);
12427 format %{ "rolq $dst, $shift" %}
12428 ins_encode %{
12429 __ rolq($dst$$Register, $shift$$constant);
12430 %}
12431 ins_pipe(ialu_reg);
12432 %}
12433
12434 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
12435 %{
12436 predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12437 match(Set dst (RotateLeft src shift));
12438 format %{ "rolxq $dst, $src, $shift" %}
12439 ins_encode %{
12440 int shift = 64 - ($shift$$constant & 63);
12441 __ rorxq($dst$$Register, $src$$Register, shift);
12442 %}
12443 ins_pipe(ialu_reg_reg);
12444 %}
12445
12446 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
12447 %{
12448 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12449 match(Set dst (RotateLeft (LoadL src) shift));
12450 ins_cost(175);
12451 format %{ "rolxq $dst, $src, $shift" %}
12452 ins_encode %{
12453 int shift = 64 - ($shift$$constant & 63);
12454 __ rorxq($dst$$Register, $src$$Address, shift);
12455 %}
12456 ins_pipe(ialu_reg_mem);
12457 %}
12458
12459 // Rotate Left by variable
12460 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12461 %{
12462 predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
12463 match(Set dst (RotateLeft dst shift));
12464 effect(KILL cr);
12465
12466 format %{ "rolq $dst, $shift" %}
12467 ins_encode %{
12468 __ rolq($dst$$Register);
12469 %}
12470 ins_pipe(ialu_reg_reg);
12471 %}
12472
12473 // Rotate Left by variable
12474 instruct rolL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
12475 %{
12476 predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
12477 match(Set dst (RotateLeft src shift));
12478 effect(KILL cr);
12479 flag(PD::Flag_ndd_demotable_opr1);
12480
12481 format %{ "erolq $dst, $src, $shift\t# rotate left(long ndd)" %}
12482 ins_encode %{
12483 __ erolq($dst$$Register, $src$$Register, false);
12484 %}
12485 ins_pipe(ialu_reg_reg);
12486 %}
12487
12488 // Rotate Right by constant.
12489 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
12490 %{
12491 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12492 match(Set dst (RotateRight dst shift));
12493 effect(KILL cr);
12494 format %{ "rorq $dst, $shift" %}
12495 ins_encode %{
12496 __ rorq($dst$$Register, $shift$$constant);
12497 %}
12498 ins_pipe(ialu_reg);
12499 %}
12500
12501 // Rotate Right by constant
12502 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
12503 %{
12504 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12505 match(Set dst (RotateRight src shift));
12506 format %{ "rorxq $dst, $src, $shift" %}
12507 ins_encode %{
12508 __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
12509 %}
12510 ins_pipe(ialu_reg_reg);
12511 %}
12512
12513 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
12514 %{
12515 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12516 match(Set dst (RotateRight (LoadL src) shift));
12517 ins_cost(175);
12518 format %{ "rorxq $dst, $src, $shift" %}
12519 ins_encode %{
12520 __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
12521 %}
12522 ins_pipe(ialu_reg_mem);
12523 %}
12524
12525 // Rotate Right by variable
12526 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12527 %{
12528 predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
12529 match(Set dst (RotateRight dst shift));
12530 effect(KILL cr);
12531 format %{ "rorq $dst, $shift" %}
12532 ins_encode %{
12533 __ rorq($dst$$Register);
12534 %}
12535 ins_pipe(ialu_reg_reg);
12536 %}
12537
12538 // Rotate Right by variable
12539 instruct rorL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
12540 %{
12541 predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
12542 match(Set dst (RotateRight src shift));
12543 effect(KILL cr);
12544 flag(PD::Flag_ndd_demotable_opr1);
12545
12546 format %{ "erorq $dst, $src, $shift\t# rotate right(long ndd)" %}
12547 ins_encode %{
12548 __ erorq($dst$$Register, $src$$Register, false);
12549 %}
12550 ins_pipe(ialu_reg_reg);
12551 %}
12552
12553 //----------------------------- CompressBits/ExpandBits ------------------------
12554
12555 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
12556 predicate(n->bottom_type()->isa_long());
12557 match(Set dst (CompressBits src mask));
12558 format %{ "pextq $dst, $src, $mask\t! parallel bit extract" %}
12559 ins_encode %{
12560 __ pextq($dst$$Register, $src$$Register, $mask$$Register);
12561 %}
12562 ins_pipe( pipe_slow );
12563 %}
12564
12565 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
12566 predicate(n->bottom_type()->isa_long());
12567 match(Set dst (ExpandBits src mask));
12568 format %{ "pdepq $dst, $src, $mask\t! parallel bit deposit" %}
12569 ins_encode %{
12570 __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
12571 %}
12572 ins_pipe( pipe_slow );
12573 %}
12574
12575 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
12576 predicate(n->bottom_type()->isa_long());
12577 match(Set dst (CompressBits src (LoadL mask)));
12578 format %{ "pextq $dst, $src, $mask\t! parallel bit extract" %}
12579 ins_encode %{
12580 __ pextq($dst$$Register, $src$$Register, $mask$$Address);
12581 %}
12582 ins_pipe( pipe_slow );
12583 %}
12584
12585 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
12586 predicate(n->bottom_type()->isa_long());
12587 match(Set dst (ExpandBits src (LoadL mask)));
12588 format %{ "pdepq $dst, $src, $mask\t! parallel bit deposit" %}
12589 ins_encode %{
12590 __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
12591 %}
12592 ins_pipe( pipe_slow );
12593 %}
12594
12595
12596 // Logical Instructions
12597
12598 // Integer Logical Instructions
12599
12600 // And Instructions
12601 // And Register with Register
12602 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
12603 %{
12604 predicate(!UseAPX);
12605 match(Set dst (AndI dst src));
12606 effect(KILL cr);
12607 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12608
12609 format %{ "andl $dst, $src\t# int" %}
12610 ins_encode %{
12611 __ andl($dst$$Register, $src$$Register);
12612 %}
12613 ins_pipe(ialu_reg_reg);
12614 %}
12615
12616 // And Register with Register using New Data Destination (NDD)
12617 instruct andI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
12618 %{
12619 predicate(UseAPX);
12620 match(Set dst (AndI src1 src2));
12621 effect(KILL cr);
12622 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
12623
12624 format %{ "eandl $dst, $src1, $src2\t# int ndd" %}
12625 ins_encode %{
12626 __ eandl($dst$$Register, $src1$$Register, $src2$$Register, false);
12627
12628 %}
12629 ins_pipe(ialu_reg_reg);
12630 %}
12631
12632 // And Register with Immediate 255
12633 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
12634 %{
12635 match(Set dst (AndI src mask));
12636
12637 format %{ "movzbl $dst, $src\t# int & 0xFF" %}
12638 ins_encode %{
12639 __ movzbl($dst$$Register, $src$$Register);
12640 %}
12641 ins_pipe(ialu_reg);
12642 %}
12643
12644 // And Register with Immediate 255 and promote to long
12645 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
12646 %{
12647 match(Set dst (ConvI2L (AndI src mask)));
12648
12649 format %{ "movzbl $dst, $src\t# int & 0xFF -> long" %}
12650 ins_encode %{
12651 __ movzbl($dst$$Register, $src$$Register);
12652 %}
12653 ins_pipe(ialu_reg);
12654 %}
12655
12656 // And Register with Immediate 65535
12657 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
12658 %{
12659 match(Set dst (AndI src mask));
12660
12661 format %{ "movzwl $dst, $src\t# int & 0xFFFF" %}
12662 ins_encode %{
12663 __ movzwl($dst$$Register, $src$$Register);
12664 %}
12665 ins_pipe(ialu_reg);
12666 %}
12667
12668 // And Register with Immediate 65535 and promote to long
12669 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
12670 %{
12671 match(Set dst (ConvI2L (AndI src mask)));
12672
12673 format %{ "movzwl $dst, $src\t# int & 0xFFFF -> long" %}
12674 ins_encode %{
12675 __ movzwl($dst$$Register, $src$$Register);
12676 %}
12677 ins_pipe(ialu_reg);
12678 %}
12679
12680 // Can skip int2long conversions after AND with small bitmask
12681 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src, immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
12682 %{
12683 predicate(VM_Version::supports_bmi2());
12684 ins_cost(125);
12685 effect(TEMP tmp, KILL cr);
12686 match(Set dst (ConvI2L (AndI src mask)));
12687 format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int & immI_Pow2M1 -> long" %}
12688 ins_encode %{
12689 __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
12690 __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
12691 %}
12692 ins_pipe(ialu_reg_reg);
12693 %}
12694
12695 // And Register with Immediate
12696 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
12697 %{
12698 predicate(!UseAPX);
12699 match(Set dst (AndI dst src));
12700 effect(KILL cr);
12701 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12702
12703 format %{ "andl $dst, $src\t# int" %}
12704 ins_encode %{
12705 __ andl($dst$$Register, $src$$constant);
12706 %}
12707 ins_pipe(ialu_reg);
12708 %}
12709
12710 instruct andI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
12711 %{
12712 predicate(UseAPX);
12713 match(Set dst (AndI src1 src2));
12714 effect(KILL cr);
12715 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
12716
12717 format %{ "eandl $dst, $src1, $src2\t# int ndd" %}
12718 ins_encode %{
12719 __ eandl($dst$$Register, $src1$$Register, $src2$$constant, false);
12720 %}
12721 ins_pipe(ialu_reg);
12722 %}
12723
12724 // And Register with Memory
12725 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
12726 %{
12727 match(Set dst (AndI dst (LoadI src)));
12728 effect(KILL cr);
12729 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12730
12731 ins_cost(150);
12732 format %{ "andl $dst, $src\t# int" %}
12733 ins_encode %{
12734 __ andl($dst$$Register, $src$$Address);
12735 %}
12736 ins_pipe(ialu_reg_mem);
12737 %}
12738
12739 // And Memory with Register
12740 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
12741 %{
12742 match(Set dst (StoreB dst (AndI (LoadB dst) src)));
12743 effect(KILL cr);
12744 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12745
12746 ins_cost(150);
12747 format %{ "andb $dst, $src\t# byte" %}
12748 ins_encode %{
12749 __ andb($dst$$Address, $src$$Register);
12750 %}
12751 ins_pipe(ialu_mem_reg);
12752 %}
12753
12754 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
12755 %{
12756 match(Set dst (StoreI dst (AndI (LoadI dst) src)));
12757 effect(KILL cr);
12758 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12759
12760 ins_cost(150);
12761 format %{ "andl $dst, $src\t# int" %}
12762 ins_encode %{
12763 __ andl($dst$$Address, $src$$Register);
12764 %}
12765 ins_pipe(ialu_mem_reg);
12766 %}
12767
12768 // And Memory with Immediate
12769 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
12770 %{
12771 match(Set dst (StoreI dst (AndI (LoadI dst) src)));
12772 effect(KILL cr);
12773 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12774
12775 ins_cost(125);
12776 format %{ "andl $dst, $src\t# int" %}
12777 ins_encode %{
12778 __ andl($dst$$Address, $src$$constant);
12779 %}
12780 ins_pipe(ialu_mem_imm);
12781 %}
12782
12783 // BMI1 instructions
12784 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
12785 match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
12786 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12787 effect(KILL cr);
12788 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12789
12790 ins_cost(125);
12791 format %{ "andnl $dst, $src1, $src2" %}
12792
12793 ins_encode %{
12794 __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
12795 %}
12796 ins_pipe(ialu_reg_mem);
12797 %}
12798
12799 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
12800 match(Set dst (AndI (XorI src1 minus_1) src2));
12801 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12802 effect(KILL cr);
12803 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12804
12805 format %{ "andnl $dst, $src1, $src2" %}
12806
12807 ins_encode %{
12808 __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
12809 %}
12810 ins_pipe(ialu_reg);
12811 %}
12812
12813 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
12814 match(Set dst (AndI (SubI imm_zero src) src));
12815 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12816 effect(KILL cr);
12817 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12818
12819 format %{ "blsil $dst, $src" %}
12820
12821 ins_encode %{
12822 __ blsil($dst$$Register, $src$$Register);
12823 %}
12824 ins_pipe(ialu_reg);
12825 %}
12826
12827 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
12828 match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
12829 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12830 effect(KILL cr);
12831 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12832
12833 ins_cost(125);
12834 format %{ "blsil $dst, $src" %}
12835
12836 ins_encode %{
12837 __ blsil($dst$$Register, $src$$Address);
12838 %}
12839 ins_pipe(ialu_reg_mem);
12840 %}
12841
12842 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
12843 %{
12844 match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
12845 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12846 effect(KILL cr);
12847 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
12848
12849 ins_cost(125);
12850 format %{ "blsmskl $dst, $src" %}
12851
12852 ins_encode %{
12853 __ blsmskl($dst$$Register, $src$$Address);
12854 %}
12855 ins_pipe(ialu_reg_mem);
12856 %}
12857
12858 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
12859 %{
12860 match(Set dst (XorI (AddI src minus_1) src));
12861 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12862 effect(KILL cr);
12863 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
12864
12865 format %{ "blsmskl $dst, $src" %}
12866
12867 ins_encode %{
12868 __ blsmskl($dst$$Register, $src$$Register);
12869 %}
12870
12871 ins_pipe(ialu_reg);
12872 %}
12873
12874 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
12875 %{
12876 match(Set dst (AndI (AddI src minus_1) src) );
12877 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12878 effect(KILL cr);
12879 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12880
12881 format %{ "blsrl $dst, $src" %}
12882
12883 ins_encode %{
12884 __ blsrl($dst$$Register, $src$$Register);
12885 %}
12886
12887 ins_pipe(ialu_reg_mem);
12888 %}
12889
12890 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
12891 %{
12892 match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
12893 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12894 effect(KILL cr);
12895 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12896
12897 ins_cost(125);
12898 format %{ "blsrl $dst, $src" %}
12899
12900 ins_encode %{
12901 __ blsrl($dst$$Register, $src$$Address);
12902 %}
12903
12904 ins_pipe(ialu_reg);
12905 %}
12906
12907 // Or Instructions
12908 // Or Register with Register
12909 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
12910 %{
12911 predicate(!UseAPX);
12912 match(Set dst (OrI dst src));
12913 effect(KILL cr);
12914 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12915
12916 format %{ "orl $dst, $src\t# int" %}
12917 ins_encode %{
12918 __ orl($dst$$Register, $src$$Register);
12919 %}
12920 ins_pipe(ialu_reg_reg);
12921 %}
12922
12923 // Or Register with Register using New Data Destination (NDD)
12924 instruct orI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
12925 %{
12926 predicate(UseAPX);
12927 match(Set dst (OrI src1 src2));
12928 effect(KILL cr);
12929 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
12930
12931 format %{ "eorl $dst, $src1, $src2\t# int ndd" %}
12932 ins_encode %{
12933 __ eorl($dst$$Register, $src1$$Register, $src2$$Register, false);
12934 %}
12935 ins_pipe(ialu_reg_reg);
12936 %}
12937
12938 // Or Register with Immediate
12939 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
12940 %{
12941 predicate(!UseAPX);
12942 match(Set dst (OrI dst src));
12943 effect(KILL cr);
12944 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12945
12946 format %{ "orl $dst, $src\t# int" %}
12947 ins_encode %{
12948 __ orl($dst$$Register, $src$$constant);
12949 %}
12950 ins_pipe(ialu_reg);
12951 %}
12952
12953 instruct orI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
12954 %{
12955 predicate(UseAPX);
12956 match(Set dst (OrI src1 src2));
12957 effect(KILL cr);
12958 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
12959
12960 format %{ "eorl $dst, $src1, $src2\t# int ndd" %}
12961 ins_encode %{
12962 __ eorl($dst$$Register, $src1$$Register, $src2$$constant, false);
12963 %}
12964 ins_pipe(ialu_reg);
12965 %}
12966
12967 instruct orI_rReg_imm_rReg_ndd(rRegI dst, immI src1, rRegI src2, rFlagsReg cr)
12968 %{
12969 predicate(UseAPX);
12970 match(Set dst (OrI src1 src2));
12971 effect(KILL cr);
12972 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
12973
12974 format %{ "eorl $dst, $src2, $src1\t# int ndd" %}
12975 ins_encode %{
12976 __ eorl($dst$$Register, $src2$$Register, $src1$$constant, false);
12977 %}
12978 ins_pipe(ialu_reg);
12979 %}
12980
12981 // Or Register with Memory
12982 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
12983 %{
12984 match(Set dst (OrI dst (LoadI src)));
12985 effect(KILL cr);
12986 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12987
12988 ins_cost(150);
12989 format %{ "orl $dst, $src\t# int" %}
12990 ins_encode %{
12991 __ orl($dst$$Register, $src$$Address);
12992 %}
12993 ins_pipe(ialu_reg_mem);
12994 %}
12995
12996 // Or Memory with Register
12997 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
12998 %{
12999 match(Set dst (StoreB dst (OrI (LoadB dst) src)));
13000 effect(KILL cr);
13001 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13002
13003 ins_cost(150);
13004 format %{ "orb $dst, $src\t# byte" %}
13005 ins_encode %{
13006 __ orb($dst$$Address, $src$$Register);
13007 %}
13008 ins_pipe(ialu_mem_reg);
13009 %}
13010
13011 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
13012 %{
13013 match(Set dst (StoreI dst (OrI (LoadI dst) src)));
13014 effect(KILL cr);
13015 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13016
13017 ins_cost(150);
13018 format %{ "orl $dst, $src\t# int" %}
13019 ins_encode %{
13020 __ orl($dst$$Address, $src$$Register);
13021 %}
13022 ins_pipe(ialu_mem_reg);
13023 %}
13024
13025 // Or Memory with Immediate
13026 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
13027 %{
13028 match(Set dst (StoreI dst (OrI (LoadI dst) src)));
13029 effect(KILL cr);
13030 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13031
13032 ins_cost(125);
13033 format %{ "orl $dst, $src\t# int" %}
13034 ins_encode %{
13035 __ orl($dst$$Address, $src$$constant);
13036 %}
13037 ins_pipe(ialu_mem_imm);
13038 %}
13039
13040 // Xor Instructions
13041 // Xor Register with Register
13042 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
13043 %{
13044 predicate(!UseAPX);
13045 match(Set dst (XorI dst src));
13046 effect(KILL cr);
13047 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13048
13049 format %{ "xorl $dst, $src\t# int" %}
13050 ins_encode %{
13051 __ xorl($dst$$Register, $src$$Register);
13052 %}
13053 ins_pipe(ialu_reg_reg);
13054 %}
13055
13056 // Xor Register with Register using New Data Destination (NDD)
13057 instruct xorI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13058 %{
13059 predicate(UseAPX);
13060 match(Set dst (XorI src1 src2));
13061 effect(KILL cr);
13062 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13063
13064 format %{ "exorl $dst, $src1, $src2\t# int ndd" %}
13065 ins_encode %{
13066 __ exorl($dst$$Register, $src1$$Register, $src2$$Register, false);
13067 %}
13068 ins_pipe(ialu_reg_reg);
13069 %}
13070
13071 // Xor Register with Immediate -1
13072 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm)
13073 %{
13074 predicate(!UseAPX);
13075 match(Set dst (XorI dst imm));
13076
13077 format %{ "notl $dst" %}
13078 ins_encode %{
13079 __ notl($dst$$Register);
13080 %}
13081 ins_pipe(ialu_reg);
13082 %}
13083
13084 instruct xorI_rReg_im1_ndd(rRegI dst, rRegI src, immI_M1 imm)
13085 %{
13086 match(Set dst (XorI src imm));
13087 predicate(UseAPX);
13088 flag(PD::Flag_ndd_demotable_opr1);
13089
13090 format %{ "enotl $dst, $src" %}
13091 ins_encode %{
13092 __ enotl($dst$$Register, $src$$Register);
13093 %}
13094 ins_pipe(ialu_reg);
13095 %}
13096
13097 // Xor Register with Immediate
13098 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
13099 %{
13100 // Strict predicate check to make selection of xorI_rReg_im1 cost agnostic if immI src is -1.
13101 predicate(!UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
13102 match(Set dst (XorI dst src));
13103 effect(KILL cr);
13104 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13105
13106 format %{ "xorl $dst, $src\t# int" %}
13107 ins_encode %{
13108 __ xorl($dst$$Register, $src$$constant);
13109 %}
13110 ins_pipe(ialu_reg);
13111 %}
13112
13113 instruct xorI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
13114 %{
13115 // Strict predicate check to make selection of xorI_rReg_im1_ndd cost agnostic if immI src2 is -1.
13116 predicate(UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
13117 match(Set dst (XorI src1 src2));
13118 effect(KILL cr);
13119 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13120
13121 format %{ "exorl $dst, $src1, $src2\t# int ndd" %}
13122 ins_encode %{
13123 __ exorl($dst$$Register, $src1$$Register, $src2$$constant, false);
13124 %}
13125 ins_pipe(ialu_reg);
13126 %}
13127
13128 // Xor Register with Memory
13129 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
13130 %{
13131 match(Set dst (XorI dst (LoadI src)));
13132 effect(KILL cr);
13133 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13134
13135 ins_cost(150);
13136 format %{ "xorl $dst, $src\t# int" %}
13137 ins_encode %{
13138 __ xorl($dst$$Register, $src$$Address);
13139 %}
13140 ins_pipe(ialu_reg_mem);
13141 %}
13142
13143 // Xor Memory with Register
13144 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
13145 %{
13146 match(Set dst (StoreB dst (XorI (LoadB dst) src)));
13147 effect(KILL cr);
13148 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13149
13150 ins_cost(150);
13151 format %{ "xorb $dst, $src\t# byte" %}
13152 ins_encode %{
13153 __ xorb($dst$$Address, $src$$Register);
13154 %}
13155 ins_pipe(ialu_mem_reg);
13156 %}
13157
13158 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
13159 %{
13160 match(Set dst (StoreI dst (XorI (LoadI dst) src)));
13161 effect(KILL cr);
13162 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13163
13164 ins_cost(150);
13165 format %{ "xorl $dst, $src\t# int" %}
13166 ins_encode %{
13167 __ xorl($dst$$Address, $src$$Register);
13168 %}
13169 ins_pipe(ialu_mem_reg);
13170 %}
13171
13172 // Xor Memory with Immediate
13173 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
13174 %{
13175 match(Set dst (StoreI dst (XorI (LoadI dst) src)));
13176 effect(KILL cr);
13177 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13178
13179 ins_cost(125);
13180 format %{ "xorl $dst, $src\t# int" %}
13181 ins_encode %{
13182 __ xorl($dst$$Address, $src$$constant);
13183 %}
13184 ins_pipe(ialu_mem_imm);
13185 %}
13186
13187
13188 // Long Logical Instructions
13189
13190 // And Instructions
13191 // And Register with Register
13192 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
13193 %{
13194 predicate(!UseAPX);
13195 match(Set dst (AndL dst src));
13196 effect(KILL cr);
13197 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13198
13199 format %{ "andq $dst, $src\t# long" %}
13200 ins_encode %{
13201 __ andq($dst$$Register, $src$$Register);
13202 %}
13203 ins_pipe(ialu_reg_reg);
13204 %}
13205
13206 // And Register with Register using New Data Destination (NDD)
13207 instruct andL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
13208 %{
13209 predicate(UseAPX);
13210 match(Set dst (AndL src1 src2));
13211 effect(KILL cr);
13212 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13213
13214 format %{ "eandq $dst, $src1, $src2\t# long ndd" %}
13215 ins_encode %{
13216 __ eandq($dst$$Register, $src1$$Register, $src2$$Register, false);
13217
13218 %}
13219 ins_pipe(ialu_reg_reg);
13220 %}
13221
13222 // And Register with Immediate 255
13223 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
13224 %{
13225 match(Set dst (AndL src mask));
13226
13227 format %{ "movzbl $dst, $src\t# long & 0xFF" %}
13228 ins_encode %{
13229 // movzbl zeroes out the upper 32-bit and does not need REX.W
13230 __ movzbl($dst$$Register, $src$$Register);
13231 %}
13232 ins_pipe(ialu_reg);
13233 %}
13234
13235 // And Register with Immediate 65535
13236 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
13237 %{
13238 match(Set dst (AndL src mask));
13239
13240 format %{ "movzwl $dst, $src\t# long & 0xFFFF" %}
13241 ins_encode %{
13242 // movzwl zeroes out the upper 32-bit and does not need REX.W
13243 __ movzwl($dst$$Register, $src$$Register);
13244 %}
13245 ins_pipe(ialu_reg);
13246 %}
13247
13248 // And Register with Immediate
13249 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
13250 %{
13251 predicate(!UseAPX);
13252 match(Set dst (AndL dst src));
13253 effect(KILL cr);
13254 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13255
13256 format %{ "andq $dst, $src\t# long" %}
13257 ins_encode %{
13258 __ andq($dst$$Register, $src$$constant);
13259 %}
13260 ins_pipe(ialu_reg);
13261 %}
13262
13263 instruct andL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
13264 %{
13265 predicate(UseAPX);
13266 match(Set dst (AndL src1 src2));
13267 effect(KILL cr);
13268 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13269
13270 format %{ "eandq $dst, $src1, $src2\t# long ndd" %}
13271 ins_encode %{
13272 __ eandq($dst$$Register, $src1$$Register, $src2$$constant, false);
13273 %}
13274 ins_pipe(ialu_reg);
13275 %}
13276
13277 // And Register with Memory
13278 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
13279 %{
13280 match(Set dst (AndL dst (LoadL src)));
13281 effect(KILL cr);
13282 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13283
13284 ins_cost(150);
13285 format %{ "andq $dst, $src\t# long" %}
13286 ins_encode %{
13287 __ andq($dst$$Register, $src$$Address);
13288 %}
13289 ins_pipe(ialu_reg_mem);
13290 %}
13291
13292 // And Memory with Register
13293 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
13294 %{
13295 match(Set dst (StoreL dst (AndL (LoadL dst) src)));
13296 effect(KILL cr);
13297 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13298
13299 ins_cost(150);
13300 format %{ "andq $dst, $src\t# long" %}
13301 ins_encode %{
13302 __ andq($dst$$Address, $src$$Register);
13303 %}
13304 ins_pipe(ialu_mem_reg);
13305 %}
13306
13307 // And Memory with Immediate
13308 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
13309 %{
13310 match(Set dst (StoreL dst (AndL (LoadL dst) src)));
13311 effect(KILL cr);
13312 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13313
13314 ins_cost(125);
13315 format %{ "andq $dst, $src\t# long" %}
13316 ins_encode %{
13317 __ andq($dst$$Address, $src$$constant);
13318 %}
13319 ins_pipe(ialu_mem_imm);
13320 %}
13321
13322 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
13323 %{
13324 // con should be a pure 64-bit immediate given that not(con) is a power of 2
13325 // because AND/OR works well enough for 8/32-bit values.
13326 predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
13327
13328 match(Set dst (StoreL dst (AndL (LoadL dst) con)));
13329 effect(KILL cr);
13330
13331 ins_cost(125);
13332 format %{ "btrq $dst, log2(not($con))\t# long" %}
13333 ins_encode %{
13334 __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
13335 %}
13336 ins_pipe(ialu_mem_imm);
13337 %}
13338
13339 // BMI1 instructions
13340 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
13341 match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
13342 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13343 effect(KILL cr);
13344 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13345
13346 ins_cost(125);
13347 format %{ "andnq $dst, $src1, $src2" %}
13348
13349 ins_encode %{
13350 __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
13351 %}
13352 ins_pipe(ialu_reg_mem);
13353 %}
13354
13355 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
13356 match(Set dst (AndL (XorL src1 minus_1) src2));
13357 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13358 effect(KILL cr);
13359 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13360
13361 format %{ "andnq $dst, $src1, $src2" %}
13362
13363 ins_encode %{
13364 __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
13365 %}
13366 ins_pipe(ialu_reg_mem);
13367 %}
13368
13369 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
13370 match(Set dst (AndL (SubL imm_zero src) src));
13371 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13372 effect(KILL cr);
13373 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13374
13375 format %{ "blsiq $dst, $src" %}
13376
13377 ins_encode %{
13378 __ blsiq($dst$$Register, $src$$Register);
13379 %}
13380 ins_pipe(ialu_reg);
13381 %}
13382
13383 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
13384 match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
13385 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13386 effect(KILL cr);
13387 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13388
13389 ins_cost(125);
13390 format %{ "blsiq $dst, $src" %}
13391
13392 ins_encode %{
13393 __ blsiq($dst$$Register, $src$$Address);
13394 %}
13395 ins_pipe(ialu_reg_mem);
13396 %}
13397
13398 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
13399 %{
13400 match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
13401 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13402 effect(KILL cr);
13403 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
13404
13405 ins_cost(125);
13406 format %{ "blsmskq $dst, $src" %}
13407
13408 ins_encode %{
13409 __ blsmskq($dst$$Register, $src$$Address);
13410 %}
13411 ins_pipe(ialu_reg_mem);
13412 %}
13413
13414 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
13415 %{
13416 match(Set dst (XorL (AddL src minus_1) src));
13417 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13418 effect(KILL cr);
13419 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
13420
13421 format %{ "blsmskq $dst, $src" %}
13422
13423 ins_encode %{
13424 __ blsmskq($dst$$Register, $src$$Register);
13425 %}
13426
13427 ins_pipe(ialu_reg);
13428 %}
13429
13430 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
13431 %{
13432 match(Set dst (AndL (AddL src minus_1) src) );
13433 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13434 effect(KILL cr);
13435 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13436
13437 format %{ "blsrq $dst, $src" %}
13438
13439 ins_encode %{
13440 __ blsrq($dst$$Register, $src$$Register);
13441 %}
13442
13443 ins_pipe(ialu_reg);
13444 %}
13445
13446 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
13447 %{
13448 match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
13449 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13450 effect(KILL cr);
13451 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13452
13453 ins_cost(125);
13454 format %{ "blsrq $dst, $src" %}
13455
13456 ins_encode %{
13457 __ blsrq($dst$$Register, $src$$Address);
13458 %}
13459
13460 ins_pipe(ialu_reg);
13461 %}
13462
13463 // Or Instructions
13464 // Or Register with Register
13465 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
13466 %{
13467 predicate(!UseAPX);
13468 match(Set dst (OrL dst src));
13469 effect(KILL cr);
13470 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13471
13472 format %{ "orq $dst, $src\t# long" %}
13473 ins_encode %{
13474 __ orq($dst$$Register, $src$$Register);
13475 %}
13476 ins_pipe(ialu_reg_reg);
13477 %}
13478
13479 // Or Register with Register using New Data Destination (NDD)
13480 instruct orL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
13481 %{
13482 predicate(UseAPX);
13483 match(Set dst (OrL src1 src2));
13484 effect(KILL cr);
13485 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13486
13487 format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
13488 ins_encode %{
13489 __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
13490
13491 %}
13492 ins_pipe(ialu_reg_reg);
13493 %}
13494
13495 // Use any_RegP to match R15 (TLS register) without spilling.
13496 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
13497 predicate(!UseAPX);
13498 match(Set dst (OrL dst (CastP2X src)));
13499 effect(KILL cr);
13500 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13501
13502 format %{ "orq $dst, $src\t# long" %}
13503 ins_encode %{
13504 __ orq($dst$$Register, $src$$Register);
13505 %}
13506 ins_pipe(ialu_reg_reg);
13507 %}
13508
13509 instruct orL_rReg_castP2X_ndd(rRegL dst, any_RegP src1, any_RegP src2, rFlagsReg cr) %{
13510 predicate(UseAPX);
13511 match(Set dst (OrL src1 (CastP2X src2)));
13512 effect(KILL cr);
13513 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13514
13515 format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
13516 ins_encode %{
13517 __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
13518 %}
13519 ins_pipe(ialu_reg_reg);
13520 %}
13521
13522 // Or Register with Immediate
13523 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
13524 %{
13525 predicate(!UseAPX);
13526 match(Set dst (OrL dst src));
13527 effect(KILL cr);
13528 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13529
13530 format %{ "orq $dst, $src\t# long" %}
13531 ins_encode %{
13532 __ orq($dst$$Register, $src$$constant);
13533 %}
13534 ins_pipe(ialu_reg);
13535 %}
13536
13537 instruct orL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
13538 %{
13539 predicate(UseAPX);
13540 match(Set dst (OrL src1 src2));
13541 effect(KILL cr);
13542 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13543
13544 format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
13545 ins_encode %{
13546 __ eorq($dst$$Register, $src1$$Register, $src2$$constant, false);
13547 %}
13548 ins_pipe(ialu_reg);
13549 %}
13550
13551 instruct orL_rReg_imm_rReg_ndd(rRegL dst, immL32 src1, rRegL src2, rFlagsReg cr)
13552 %{
13553 predicate(UseAPX);
13554 match(Set dst (OrL src1 src2));
13555 effect(KILL cr);
13556 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13557
13558 format %{ "eorq $dst, $src2, $src1\t# long ndd" %}
13559 ins_encode %{
13560 __ eorq($dst$$Register, $src2$$Register, $src1$$constant, false);
13561 %}
13562 ins_pipe(ialu_reg);
13563 %}
13564
13565 // Or Register with Memory
13566 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
13567 %{
13568 match(Set dst (OrL dst (LoadL src)));
13569 effect(KILL cr);
13570 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13571
13572 ins_cost(150);
13573 format %{ "orq $dst, $src\t# long" %}
13574 ins_encode %{
13575 __ orq($dst$$Register, $src$$Address);
13576 %}
13577 ins_pipe(ialu_reg_mem);
13578 %}
13579
13580 // Or Memory with Register
13581 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
13582 %{
13583 match(Set dst (StoreL dst (OrL (LoadL dst) src)));
13584 effect(KILL cr);
13585 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13586
13587 ins_cost(150);
13588 format %{ "orq $dst, $src\t# long" %}
13589 ins_encode %{
13590 __ orq($dst$$Address, $src$$Register);
13591 %}
13592 ins_pipe(ialu_mem_reg);
13593 %}
13594
13595 // Or Memory with Immediate
13596 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
13597 %{
13598 match(Set dst (StoreL dst (OrL (LoadL dst) src)));
13599 effect(KILL cr);
13600 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13601
13602 ins_cost(125);
13603 format %{ "orq $dst, $src\t# long" %}
13604 ins_encode %{
13605 __ orq($dst$$Address, $src$$constant);
13606 %}
13607 ins_pipe(ialu_mem_imm);
13608 %}
13609
13610 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
13611 %{
13612 // con should be a pure 64-bit power of 2 immediate
13613 // because AND/OR works well enough for 8/32-bit values.
13614 predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
13615
13616 match(Set dst (StoreL dst (OrL (LoadL dst) con)));
13617 effect(KILL cr);
13618
13619 ins_cost(125);
13620 format %{ "btsq $dst, log2($con)\t# long" %}
13621 ins_encode %{
13622 __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
13623 %}
13624 ins_pipe(ialu_mem_imm);
13625 %}
13626
13627 // Xor Instructions
13628 // Xor Register with Register
13629 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
13630 %{
13631 predicate(!UseAPX);
13632 match(Set dst (XorL dst src));
13633 effect(KILL cr);
13634 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13635
13636 format %{ "xorq $dst, $src\t# long" %}
13637 ins_encode %{
13638 __ xorq($dst$$Register, $src$$Register);
13639 %}
13640 ins_pipe(ialu_reg_reg);
13641 %}
13642
13643 // Xor Register with Register using New Data Destination (NDD)
13644 instruct xorL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
13645 %{
13646 predicate(UseAPX);
13647 match(Set dst (XorL src1 src2));
13648 effect(KILL cr);
13649 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13650
13651 format %{ "exorq $dst, $src1, $src2\t# long ndd" %}
13652 ins_encode %{
13653 __ exorq($dst$$Register, $src1$$Register, $src2$$Register, false);
13654 %}
13655 ins_pipe(ialu_reg_reg);
13656 %}
13657
13658 // Xor Register with Immediate -1
13659 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm)
13660 %{
13661 predicate(!UseAPX);
13662 match(Set dst (XorL dst imm));
13663
13664 format %{ "notq $dst" %}
13665 ins_encode %{
13666 __ notq($dst$$Register);
13667 %}
13668 ins_pipe(ialu_reg);
13669 %}
13670
13671 instruct xorL_rReg_im1_ndd(rRegL dst,rRegL src, immL_M1 imm)
13672 %{
13673 predicate(UseAPX);
13674 match(Set dst (XorL src imm));
13675 flag(PD::Flag_ndd_demotable_opr1);
13676
13677 format %{ "enotq $dst, $src" %}
13678 ins_encode %{
13679 __ enotq($dst$$Register, $src$$Register);
13680 %}
13681 ins_pipe(ialu_reg);
13682 %}
13683
13684 // Xor Register with Immediate
13685 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
13686 %{
13687 // Strict predicate check to make selection of xorL_rReg_im1 cost agnostic if immL32 src is -1.
13688 predicate(!UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
13689 match(Set dst (XorL dst src));
13690 effect(KILL cr);
13691 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13692
13693 format %{ "xorq $dst, $src\t# long" %}
13694 ins_encode %{
13695 __ xorq($dst$$Register, $src$$constant);
13696 %}
13697 ins_pipe(ialu_reg);
13698 %}
13699
13700 instruct xorL_rReg_rReg_imm(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
13701 %{
13702 // Strict predicate check to make selection of xorL_rReg_im1_ndd cost agnostic if immL32 src2 is -1.
13703 predicate(UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
13704 match(Set dst (XorL src1 src2));
13705 effect(KILL cr);
13706 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13707
13708 format %{ "exorq $dst, $src1, $src2\t# long ndd" %}
13709 ins_encode %{
13710 __ exorq($dst$$Register, $src1$$Register, $src2$$constant, false);
13711 %}
13712 ins_pipe(ialu_reg);
13713 %}
13714
13715 // Xor Register with Memory
13716 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
13717 %{
13718 match(Set dst (XorL dst (LoadL src)));
13719 effect(KILL cr);
13720 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13721
13722 ins_cost(150);
13723 format %{ "xorq $dst, $src\t# long" %}
13724 ins_encode %{
13725 __ xorq($dst$$Register, $src$$Address);
13726 %}
13727 ins_pipe(ialu_reg_mem);
13728 %}
13729
13730 // Xor Memory with Register
13731 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
13732 %{
13733 match(Set dst (StoreL dst (XorL (LoadL dst) src)));
13734 effect(KILL cr);
13735 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13736
13737 ins_cost(150);
13738 format %{ "xorq $dst, $src\t# long" %}
13739 ins_encode %{
13740 __ xorq($dst$$Address, $src$$Register);
13741 %}
13742 ins_pipe(ialu_mem_reg);
13743 %}
13744
13745 // Xor Memory with Immediate
13746 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
13747 %{
13748 match(Set dst (StoreL dst (XorL (LoadL dst) src)));
13749 effect(KILL cr);
13750 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13751
13752 ins_cost(125);
13753 format %{ "xorq $dst, $src\t# long" %}
13754 ins_encode %{
13755 __ xorq($dst$$Address, $src$$constant);
13756 %}
13757 ins_pipe(ialu_mem_imm);
13758 %}
13759
13760 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
13761 %{
13762 match(Set dst (CmpLTMask p q));
13763 effect(KILL cr);
13764
13765 ins_cost(400);
13766 format %{ "cmpl $p, $q\t# cmpLTMask\n\t"
13767 "setcc $dst \t# emits setlt + movzbl or setzul for APX"
13768 "negl $dst" %}
13769 ins_encode %{
13770 __ cmpl($p$$Register, $q$$Register);
13771 __ setcc(Assembler::less, $dst$$Register);
13772 __ negl($dst$$Register);
13773 %}
13774 ins_pipe(pipe_slow);
13775 %}
13776
13777 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
13778 %{
13779 match(Set dst (CmpLTMask dst zero));
13780 effect(KILL cr);
13781
13782 ins_cost(100);
13783 format %{ "sarl $dst, #31\t# cmpLTMask0" %}
13784 ins_encode %{
13785 __ sarl($dst$$Register, 31);
13786 %}
13787 ins_pipe(ialu_reg);
13788 %}
13789
13790 /* Better to save a register than avoid a branch */
13791 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
13792 %{
13793 match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
13794 effect(KILL cr);
13795 ins_cost(300);
13796 format %{ "subl $p,$q\t# cadd_cmpLTMask\n\t"
13797 "jge done\n\t"
13798 "addl $p,$y\n"
13799 "done: " %}
13800 ins_encode %{
13801 Register Rp = $p$$Register;
13802 Register Rq = $q$$Register;
13803 Register Ry = $y$$Register;
13804 Label done;
13805 __ subl(Rp, Rq);
13806 __ jccb(Assembler::greaterEqual, done);
13807 __ addl(Rp, Ry);
13808 __ bind(done);
13809 %}
13810 ins_pipe(pipe_cmplt);
13811 %}
13812
13813 /* Better to save a register than avoid a branch */
13814 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
13815 %{
13816 match(Set y (AndI (CmpLTMask p q) y));
13817 effect(KILL cr);
13818
13819 ins_cost(300);
13820
13821 format %{ "cmpl $p, $q\t# and_cmpLTMask\n\t"
13822 "jlt done\n\t"
13823 "xorl $y, $y\n"
13824 "done: " %}
13825 ins_encode %{
13826 Register Rp = $p$$Register;
13827 Register Rq = $q$$Register;
13828 Register Ry = $y$$Register;
13829 Label done;
13830 __ cmpl(Rp, Rq);
13831 __ jccb(Assembler::less, done);
13832 __ xorl(Ry, Ry);
13833 __ bind(done);
13834 %}
13835 ins_pipe(pipe_cmplt);
13836 %}
13837
13838
13839 //---------- FP Instructions------------------------------------------------
13840
13841 // Really expensive, avoid
13842 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
13843 %{
13844 match(Set cr (CmpF src1 src2));
13845
13846 ins_cost(500);
13847 format %{ "ucomiss $src1, $src2\n\t"
13848 "jnp,s exit\n\t"
13849 "pushfq\t# saw NaN, set CF\n\t"
13850 "andq [rsp], #0xffffff2b\n\t"
13851 "popfq\n"
13852 "exit:" %}
13853 ins_encode %{
13854 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
13855 emit_cmpfp_fixup(masm);
13856 %}
13857 ins_pipe(pipe_slow);
13858 %}
13859
13860 instruct cmpF_cc_regCF(rFlagsRegUCF cr, regF src1, regF src2) %{
13861 match(Set cr (CmpF src1 src2));
13862
13863 ins_cost(100);
13864 format %{ "ucomiss $src1, $src2" %}
13865 ins_encode %{
13866 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
13867 %}
13868 ins_pipe(pipe_slow);
13869 %}
13870
13871 instruct cmpF_cc_regCFE(rFlagsRegUCFE cr, regF src1, regF src2) %{
13872 match(Set cr (CmpF src1 src2));
13873
13874 ins_cost(100);
13875 format %{ "evucomxss $src1, $src2" %}
13876 ins_encode %{
13877 __ evucomxss($src1$$XMMRegister, $src2$$XMMRegister);
13878 %}
13879 ins_pipe(pipe_slow);
13880 %}
13881
13882 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
13883 match(Set cr (CmpF src1 (LoadF src2)));
13884
13885 ins_cost(100);
13886 format %{ "ucomiss $src1, $src2" %}
13887 ins_encode %{
13888 __ ucomiss($src1$$XMMRegister, $src2$$Address);
13889 %}
13890 ins_pipe(pipe_slow);
13891 %}
13892
13893 instruct cmpF_cc_memCFE(rFlagsRegUCFE cr, regF src1, memory src2) %{
13894 match(Set cr (CmpF src1 (LoadF src2)));
13895
13896 ins_cost(100);
13897 format %{ "evucomxss $src1, $src2" %}
13898 ins_encode %{
13899 __ evucomxss($src1$$XMMRegister, $src2$$Address);
13900 %}
13901 ins_pipe(pipe_slow);
13902 %}
13903
13904 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
13905 match(Set cr (CmpF src con));
13906
13907 ins_cost(100);
13908 format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
13909 ins_encode %{
13910 __ ucomiss($src$$XMMRegister, $constantaddress($con));
13911 %}
13912 ins_pipe(pipe_slow);
13913 %}
13914
13915 instruct cmpF_cc_immCFE(rFlagsRegUCFE cr, regF src, immF con) %{
13916 match(Set cr (CmpF src con));
13917
13918 ins_cost(100);
13919 format %{ "evucomxss $src, [$constantaddress]\t# load from constant table: float=$con" %}
13920 ins_encode %{
13921 __ evucomxss($src$$XMMRegister, $constantaddress($con));
13922 %}
13923 ins_pipe(pipe_slow);
13924 %}
13925
13926 // Really expensive, avoid
13927 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
13928 %{
13929 match(Set cr (CmpD src1 src2));
13930
13931 ins_cost(500);
13932 format %{ "ucomisd $src1, $src2\n\t"
13933 "jnp,s exit\n\t"
13934 "pushfq\t# saw NaN, set CF\n\t"
13935 "andq [rsp], #0xffffff2b\n\t"
13936 "popfq\n"
13937 "exit:" %}
13938 ins_encode %{
13939 __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
13940 emit_cmpfp_fixup(masm);
13941 %}
13942 ins_pipe(pipe_slow);
13943 %}
13944
13945 instruct cmpD_cc_regCF(rFlagsRegUCF cr, regD src1, regD src2) %{
13946 match(Set cr (CmpD src1 src2));
13947
13948 ins_cost(100);
13949 format %{ "ucomisd $src1, $src2 test" %}
13950 ins_encode %{
13951 __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
13952 %}
13953 ins_pipe(pipe_slow);
13954 %}
13955
13956 instruct cmpD_cc_regCFE(rFlagsRegUCFE cr, regD src1, regD src2) %{
13957 match(Set cr (CmpD src1 src2));
13958
13959 ins_cost(100);
13960 format %{ "evucomxsd $src1, $src2 test" %}
13961 ins_encode %{
13962 __ evucomxsd($src1$$XMMRegister, $src2$$XMMRegister);
13963 %}
13964 ins_pipe(pipe_slow);
13965 %}
13966
13967 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
13968 match(Set cr (CmpD src1 (LoadD src2)));
13969
13970 ins_cost(100);
13971 format %{ "ucomisd $src1, $src2" %}
13972 ins_encode %{
13973 __ ucomisd($src1$$XMMRegister, $src2$$Address);
13974 %}
13975 ins_pipe(pipe_slow);
13976 %}
13977
13978 instruct cmpD_cc_memCFE(rFlagsRegUCFE cr, regD src1, memory src2) %{
13979 match(Set cr (CmpD src1 (LoadD src2)));
13980
13981 ins_cost(100);
13982 format %{ "evucomxsd $src1, $src2" %}
13983 ins_encode %{
13984 __ evucomxsd($src1$$XMMRegister, $src2$$Address);
13985 %}
13986 ins_pipe(pipe_slow);
13987 %}
13988
13989 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
13990 match(Set cr (CmpD src con));
13991 ins_cost(100);
13992 format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
13993 ins_encode %{
13994 __ ucomisd($src$$XMMRegister, $constantaddress($con));
13995 %}
13996 ins_pipe(pipe_slow);
13997 %}
13998
13999 instruct cmpD_cc_immCFE(rFlagsRegUCFE cr, regD src, immD con) %{
14000 match(Set cr (CmpD src con));
14001
14002 ins_cost(100);
14003 format %{ "evucomxsd $src, [$constantaddress]\t# load from constant table: double=$con" %}
14004 ins_encode %{
14005 __ evucomxsd($src$$XMMRegister, $constantaddress($con));
14006 %}
14007 ins_pipe(pipe_slow);
14008 %}
14009
14010 // Compare into -1,0,1
14011 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
14012 %{
14013 match(Set dst (CmpF3 src1 src2));
14014 effect(KILL cr);
14015
14016 ins_cost(275);
14017 format %{ "ucomiss $src1, $src2\n\t"
14018 "movl $dst, #-1\n\t"
14019 "jp,s done\n\t"
14020 "jb,s done\n\t"
14021 "setne $dst\n\t"
14022 "movzbl $dst, $dst\n"
14023 "done:" %}
14024 ins_encode %{
14025 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
14026 emit_cmpfp3(masm, $dst$$Register);
14027 %}
14028 ins_pipe(pipe_slow);
14029 %}
14030
14031 // Compare into -1,0,1
14032 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
14033 %{
14034 match(Set dst (CmpF3 src1 (LoadF src2)));
14035 effect(KILL cr);
14036
14037 ins_cost(275);
14038 format %{ "ucomiss $src1, $src2\n\t"
14039 "movl $dst, #-1\n\t"
14040 "jp,s done\n\t"
14041 "jb,s done\n\t"
14042 "setne $dst\n\t"
14043 "movzbl $dst, $dst\n"
14044 "done:" %}
14045 ins_encode %{
14046 __ ucomiss($src1$$XMMRegister, $src2$$Address);
14047 emit_cmpfp3(masm, $dst$$Register);
14048 %}
14049 ins_pipe(pipe_slow);
14050 %}
14051
14052 // Compare into -1,0,1
14053 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
14054 match(Set dst (CmpF3 src con));
14055 effect(KILL cr);
14056
14057 ins_cost(275);
14058 format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
14059 "movl $dst, #-1\n\t"
14060 "jp,s done\n\t"
14061 "jb,s done\n\t"
14062 "setne $dst\n\t"
14063 "movzbl $dst, $dst\n"
14064 "done:" %}
14065 ins_encode %{
14066 __ ucomiss($src$$XMMRegister, $constantaddress($con));
14067 emit_cmpfp3(masm, $dst$$Register);
14068 %}
14069 ins_pipe(pipe_slow);
14070 %}
14071
14072 // Compare into -1,0,1
14073 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
14074 %{
14075 match(Set dst (CmpD3 src1 src2));
14076 effect(KILL cr);
14077
14078 ins_cost(275);
14079 format %{ "ucomisd $src1, $src2\n\t"
14080 "movl $dst, #-1\n\t"
14081 "jp,s done\n\t"
14082 "jb,s done\n\t"
14083 "setne $dst\n\t"
14084 "movzbl $dst, $dst\n"
14085 "done:" %}
14086 ins_encode %{
14087 __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
14088 emit_cmpfp3(masm, $dst$$Register);
14089 %}
14090 ins_pipe(pipe_slow);
14091 %}
14092
14093 // Compare into -1,0,1
14094 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
14095 %{
14096 match(Set dst (CmpD3 src1 (LoadD src2)));
14097 effect(KILL cr);
14098
14099 ins_cost(275);
14100 format %{ "ucomisd $src1, $src2\n\t"
14101 "movl $dst, #-1\n\t"
14102 "jp,s done\n\t"
14103 "jb,s done\n\t"
14104 "setne $dst\n\t"
14105 "movzbl $dst, $dst\n"
14106 "done:" %}
14107 ins_encode %{
14108 __ ucomisd($src1$$XMMRegister, $src2$$Address);
14109 emit_cmpfp3(masm, $dst$$Register);
14110 %}
14111 ins_pipe(pipe_slow);
14112 %}
14113
14114 // Compare into -1,0,1
14115 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
14116 match(Set dst (CmpD3 src con));
14117 effect(KILL cr);
14118
14119 ins_cost(275);
14120 format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
14121 "movl $dst, #-1\n\t"
14122 "jp,s done\n\t"
14123 "jb,s done\n\t"
14124 "setne $dst\n\t"
14125 "movzbl $dst, $dst\n"
14126 "done:" %}
14127 ins_encode %{
14128 __ ucomisd($src$$XMMRegister, $constantaddress($con));
14129 emit_cmpfp3(masm, $dst$$Register);
14130 %}
14131 ins_pipe(pipe_slow);
14132 %}
14133
14134 //----------Arithmetic Conversion Instructions---------------------------------
14135
14136 instruct convF2D_reg_reg(regD dst, regF src)
14137 %{
14138 match(Set dst (ConvF2D src));
14139
14140 format %{ "cvtss2sd $dst, $src" %}
14141 ins_encode %{
14142 __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
14143 %}
14144 ins_pipe(pipe_slow); // XXX
14145 %}
14146
14147 instruct convF2D_reg_mem(regD dst, memory src)
14148 %{
14149 predicate(UseAVX == 0);
14150 match(Set dst (ConvF2D (LoadF src)));
14151
14152 format %{ "cvtss2sd $dst, $src" %}
14153 ins_encode %{
14154 __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
14155 %}
14156 ins_pipe(pipe_slow); // XXX
14157 %}
14158
14159 instruct convD2F_reg_reg(regF dst, regD src)
14160 %{
14161 match(Set dst (ConvD2F src));
14162
14163 format %{ "cvtsd2ss $dst, $src" %}
14164 ins_encode %{
14165 __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
14166 %}
14167 ins_pipe(pipe_slow); // XXX
14168 %}
14169
14170 instruct convD2F_reg_mem(regF dst, memory src)
14171 %{
14172 predicate(UseAVX == 0);
14173 match(Set dst (ConvD2F (LoadD src)));
14174
14175 format %{ "cvtsd2ss $dst, $src" %}
14176 ins_encode %{
14177 __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
14178 %}
14179 ins_pipe(pipe_slow); // XXX
14180 %}
14181
14182 // XXX do mem variants
14183 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
14184 %{
14185 predicate(!VM_Version::supports_avx10_2());
14186 match(Set dst (ConvF2I src));
14187 effect(KILL cr);
14188 format %{ "convert_f2i $dst, $src" %}
14189 ins_encode %{
14190 __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
14191 %}
14192 ins_pipe(pipe_slow);
14193 %}
14194
14195 instruct convF2I_reg_reg_avx10_2(rRegI dst, regF src)
14196 %{
14197 predicate(VM_Version::supports_avx10_2());
14198 match(Set dst (ConvF2I src));
14199 format %{ "evcvttss2sisl $dst, $src" %}
14200 ins_encode %{
14201 __ evcvttss2sisl($dst$$Register, $src$$XMMRegister);
14202 %}
14203 ins_pipe(pipe_slow);
14204 %}
14205
14206 instruct convF2I_reg_mem_avx10_2(rRegI dst, memory src)
14207 %{
14208 predicate(VM_Version::supports_avx10_2());
14209 match(Set dst (ConvF2I (LoadF src)));
14210 format %{ "evcvttss2sisl $dst, $src" %}
14211 ins_encode %{
14212 __ evcvttss2sisl($dst$$Register, $src$$Address);
14213 %}
14214 ins_pipe(pipe_slow);
14215 %}
14216
14217 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
14218 %{
14219 predicate(!VM_Version::supports_avx10_2());
14220 match(Set dst (ConvF2L src));
14221 effect(KILL cr);
14222 format %{ "convert_f2l $dst, $src"%}
14223 ins_encode %{
14224 __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
14225 %}
14226 ins_pipe(pipe_slow);
14227 %}
14228
14229 instruct convF2L_reg_reg_avx10_2(rRegL dst, regF src)
14230 %{
14231 predicate(VM_Version::supports_avx10_2());
14232 match(Set dst (ConvF2L src));
14233 format %{ "evcvttss2sisq $dst, $src" %}
14234 ins_encode %{
14235 __ evcvttss2sisq($dst$$Register, $src$$XMMRegister);
14236 %}
14237 ins_pipe(pipe_slow);
14238 %}
14239
14240 instruct convF2L_reg_mem_avx10_2(rRegL dst, memory src)
14241 %{
14242 predicate(VM_Version::supports_avx10_2());
14243 match(Set dst (ConvF2L (LoadF src)));
14244 format %{ "evcvttss2sisq $dst, $src" %}
14245 ins_encode %{
14246 __ evcvttss2sisq($dst$$Register, $src$$Address);
14247 %}
14248 ins_pipe(pipe_slow);
14249 %}
14250
14251 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
14252 %{
14253 predicate(!VM_Version::supports_avx10_2());
14254 match(Set dst (ConvD2I src));
14255 effect(KILL cr);
14256 format %{ "convert_d2i $dst, $src"%}
14257 ins_encode %{
14258 __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
14259 %}
14260 ins_pipe(pipe_slow);
14261 %}
14262
14263 instruct convD2I_reg_reg_avx10_2(rRegI dst, regD src)
14264 %{
14265 predicate(VM_Version::supports_avx10_2());
14266 match(Set dst (ConvD2I src));
14267 format %{ "evcvttsd2sisl $dst, $src" %}
14268 ins_encode %{
14269 __ evcvttsd2sisl($dst$$Register, $src$$XMMRegister);
14270 %}
14271 ins_pipe(pipe_slow);
14272 %}
14273
14274 instruct convD2I_reg_mem_avx10_2(rRegI dst, memory src)
14275 %{
14276 predicate(VM_Version::supports_avx10_2());
14277 match(Set dst (ConvD2I (LoadD src)));
14278 format %{ "evcvttsd2sisl $dst, $src" %}
14279 ins_encode %{
14280 __ evcvttsd2sisl($dst$$Register, $src$$Address);
14281 %}
14282 ins_pipe(pipe_slow);
14283 %}
14284
14285 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
14286 %{
14287 predicate(!VM_Version::supports_avx10_2());
14288 match(Set dst (ConvD2L src));
14289 effect(KILL cr);
14290 format %{ "convert_d2l $dst, $src"%}
14291 ins_encode %{
14292 __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
14293 %}
14294 ins_pipe(pipe_slow);
14295 %}
14296
14297 instruct convD2L_reg_reg_avx10_2(rRegL dst, regD src)
14298 %{
14299 predicate(VM_Version::supports_avx10_2());
14300 match(Set dst (ConvD2L src));
14301 format %{ "evcvttsd2sisq $dst, $src" %}
14302 ins_encode %{
14303 __ evcvttsd2sisq($dst$$Register, $src$$XMMRegister);
14304 %}
14305 ins_pipe(pipe_slow);
14306 %}
14307
14308 instruct convD2L_reg_mem_avx10_2(rRegL dst, memory src)
14309 %{
14310 predicate(VM_Version::supports_avx10_2());
14311 match(Set dst (ConvD2L (LoadD src)));
14312 format %{ "evcvttsd2sisq $dst, $src" %}
14313 ins_encode %{
14314 __ evcvttsd2sisq($dst$$Register, $src$$Address);
14315 %}
14316 ins_pipe(pipe_slow);
14317 %}
14318
14319 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
14320 %{
14321 match(Set dst (RoundD src));
14322 effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
14323 format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
14324 ins_encode %{
14325 __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
14326 %}
14327 ins_pipe(pipe_slow);
14328 %}
14329
14330 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
14331 %{
14332 match(Set dst (RoundF src));
14333 effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
14334 format %{ "round_float $dst,$src" %}
14335 ins_encode %{
14336 __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
14337 %}
14338 ins_pipe(pipe_slow);
14339 %}
14340
14341 instruct convI2F_reg_reg(vlRegF dst, rRegI src)
14342 %{
14343 predicate(!UseXmmI2F);
14344 match(Set dst (ConvI2F src));
14345
14346 format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
14347 ins_encode %{
14348 if (UseAVX > 0) {
14349 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14350 }
14351 __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
14352 %}
14353 ins_pipe(pipe_slow); // XXX
14354 %}
14355
14356 instruct convI2F_reg_mem(regF dst, memory src)
14357 %{
14358 predicate(UseAVX == 0);
14359 match(Set dst (ConvI2F (LoadI src)));
14360
14361 format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
14362 ins_encode %{
14363 __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
14364 %}
14365 ins_pipe(pipe_slow); // XXX
14366 %}
14367
14368 instruct convI2D_reg_reg(vlRegD dst, rRegI src)
14369 %{
14370 predicate(!UseXmmI2D);
14371 match(Set dst (ConvI2D src));
14372
14373 format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
14374 ins_encode %{
14375 if (UseAVX > 0) {
14376 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14377 }
14378 __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
14379 %}
14380 ins_pipe(pipe_slow); // XXX
14381 %}
14382
14383 instruct convI2D_reg_mem(regD dst, memory src)
14384 %{
14385 predicate(UseAVX == 0);
14386 match(Set dst (ConvI2D (LoadI src)));
14387
14388 format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
14389 ins_encode %{
14390 __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
14391 %}
14392 ins_pipe(pipe_slow); // XXX
14393 %}
14394
14395 instruct convXI2F_reg(regF dst, rRegI src)
14396 %{
14397 predicate(UseXmmI2F);
14398 match(Set dst (ConvI2F src));
14399
14400 format %{ "movdl $dst, $src\n\t"
14401 "cvtdq2psl $dst, $dst\t# i2f" %}
14402 ins_encode %{
14403 __ movdl($dst$$XMMRegister, $src$$Register);
14404 __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
14405 %}
14406 ins_pipe(pipe_slow); // XXX
14407 %}
14408
14409 instruct convXI2D_reg(regD dst, rRegI src)
14410 %{
14411 predicate(UseXmmI2D);
14412 match(Set dst (ConvI2D src));
14413
14414 format %{ "movdl $dst, $src\n\t"
14415 "cvtdq2pdl $dst, $dst\t# i2d" %}
14416 ins_encode %{
14417 __ movdl($dst$$XMMRegister, $src$$Register);
14418 __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
14419 %}
14420 ins_pipe(pipe_slow); // XXX
14421 %}
14422
14423 instruct convL2F_reg_reg(vlRegF dst, rRegL src)
14424 %{
14425 match(Set dst (ConvL2F src));
14426
14427 format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
14428 ins_encode %{
14429 if (UseAVX > 0) {
14430 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14431 }
14432 __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
14433 %}
14434 ins_pipe(pipe_slow); // XXX
14435 %}
14436
14437 instruct convL2F_reg_mem(regF dst, memory src)
14438 %{
14439 predicate(UseAVX == 0);
14440 match(Set dst (ConvL2F (LoadL src)));
14441
14442 format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
14443 ins_encode %{
14444 __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
14445 %}
14446 ins_pipe(pipe_slow); // XXX
14447 %}
14448
14449 instruct convL2D_reg_reg(vlRegD dst, rRegL src)
14450 %{
14451 match(Set dst (ConvL2D src));
14452
14453 format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
14454 ins_encode %{
14455 if (UseAVX > 0) {
14456 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14457 }
14458 __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
14459 %}
14460 ins_pipe(pipe_slow); // XXX
14461 %}
14462
14463 instruct convL2D_reg_mem(regD dst, memory src)
14464 %{
14465 predicate(UseAVX == 0);
14466 match(Set dst (ConvL2D (LoadL src)));
14467
14468 format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
14469 ins_encode %{
14470 __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
14471 %}
14472 ins_pipe(pipe_slow); // XXX
14473 %}
14474
14475 instruct convI2L_reg_reg(rRegL dst, rRegI src)
14476 %{
14477 match(Set dst (ConvI2L src));
14478
14479 ins_cost(125);
14480 format %{ "movslq $dst, $src\t# i2l" %}
14481 ins_encode %{
14482 __ movslq($dst$$Register, $src$$Register);
14483 %}
14484 ins_pipe(ialu_reg_reg);
14485 %}
14486
14487 // Zero-extend convert int to long
14488 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
14489 %{
14490 match(Set dst (AndL (ConvI2L src) mask));
14491
14492 format %{ "movl $dst, $src\t# i2l zero-extend\n\t" %}
14493 ins_encode %{
14494 if ($dst$$reg != $src$$reg) {
14495 __ movl($dst$$Register, $src$$Register);
14496 }
14497 %}
14498 ins_pipe(ialu_reg_reg);
14499 %}
14500
14501 // Zero-extend convert int to long
14502 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
14503 %{
14504 match(Set dst (AndL (ConvI2L (LoadI src)) mask));
14505
14506 format %{ "movl $dst, $src\t# i2l zero-extend\n\t" %}
14507 ins_encode %{
14508 __ movl($dst$$Register, $src$$Address);
14509 %}
14510 ins_pipe(ialu_reg_mem);
14511 %}
14512
14513 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
14514 %{
14515 match(Set dst (AndL src mask));
14516
14517 format %{ "movl $dst, $src\t# zero-extend long" %}
14518 ins_encode %{
14519 __ movl($dst$$Register, $src$$Register);
14520 %}
14521 ins_pipe(ialu_reg_reg);
14522 %}
14523
14524 instruct convL2I_reg_reg(rRegI dst, rRegL src)
14525 %{
14526 match(Set dst (ConvL2I src));
14527
14528 format %{ "movl $dst, $src\t# l2i" %}
14529 ins_encode %{
14530 __ movl($dst$$Register, $src$$Register);
14531 %}
14532 ins_pipe(ialu_reg_reg);
14533 %}
14534
14535
14536 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
14537 match(Set dst (MoveF2I src));
14538 effect(DEF dst, USE src);
14539
14540 ins_cost(125);
14541 format %{ "movl $dst, $src\t# MoveF2I_stack_reg" %}
14542 ins_encode %{
14543 __ movl($dst$$Register, Address(rsp, $src$$disp));
14544 %}
14545 ins_pipe(ialu_reg_mem);
14546 %}
14547
14548 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
14549 match(Set dst (MoveI2F src));
14550 effect(DEF dst, USE src);
14551
14552 ins_cost(125);
14553 format %{ "movss $dst, $src\t# MoveI2F_stack_reg" %}
14554 ins_encode %{
14555 __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
14556 %}
14557 ins_pipe(pipe_slow);
14558 %}
14559
14560 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
14561 match(Set dst (MoveD2L src));
14562 effect(DEF dst, USE src);
14563
14564 ins_cost(125);
14565 format %{ "movq $dst, $src\t# MoveD2L_stack_reg" %}
14566 ins_encode %{
14567 __ movq($dst$$Register, Address(rsp, $src$$disp));
14568 %}
14569 ins_pipe(ialu_reg_mem);
14570 %}
14571
14572 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
14573 predicate(!UseXmmLoadAndClearUpper);
14574 match(Set dst (MoveL2D src));
14575 effect(DEF dst, USE src);
14576
14577 ins_cost(125);
14578 format %{ "movlpd $dst, $src\t# MoveL2D_stack_reg" %}
14579 ins_encode %{
14580 __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
14581 %}
14582 ins_pipe(pipe_slow);
14583 %}
14584
14585 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
14586 predicate(UseXmmLoadAndClearUpper);
14587 match(Set dst (MoveL2D src));
14588 effect(DEF dst, USE src);
14589
14590 ins_cost(125);
14591 format %{ "movsd $dst, $src\t# MoveL2D_stack_reg" %}
14592 ins_encode %{
14593 __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
14594 %}
14595 ins_pipe(pipe_slow);
14596 %}
14597
14598
14599 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
14600 match(Set dst (MoveF2I src));
14601 effect(DEF dst, USE src);
14602
14603 ins_cost(95); // XXX
14604 format %{ "movss $dst, $src\t# MoveF2I_reg_stack" %}
14605 ins_encode %{
14606 __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
14607 %}
14608 ins_pipe(pipe_slow);
14609 %}
14610
14611 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
14612 match(Set dst (MoveI2F src));
14613 effect(DEF dst, USE src);
14614
14615 ins_cost(100);
14616 format %{ "movl $dst, $src\t# MoveI2F_reg_stack" %}
14617 ins_encode %{
14618 __ movl(Address(rsp, $dst$$disp), $src$$Register);
14619 %}
14620 ins_pipe( ialu_mem_reg );
14621 %}
14622
14623 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
14624 match(Set dst (MoveD2L src));
14625 effect(DEF dst, USE src);
14626
14627 ins_cost(95); // XXX
14628 format %{ "movsd $dst, $src\t# MoveL2D_reg_stack" %}
14629 ins_encode %{
14630 __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
14631 %}
14632 ins_pipe(pipe_slow);
14633 %}
14634
14635 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
14636 match(Set dst (MoveL2D src));
14637 effect(DEF dst, USE src);
14638
14639 ins_cost(100);
14640 format %{ "movq $dst, $src\t# MoveL2D_reg_stack" %}
14641 ins_encode %{
14642 __ movq(Address(rsp, $dst$$disp), $src$$Register);
14643 %}
14644 ins_pipe(ialu_mem_reg);
14645 %}
14646
14647 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
14648 match(Set dst (MoveF2I src));
14649 effect(DEF dst, USE src);
14650 ins_cost(85);
14651 format %{ "movd $dst,$src\t# MoveF2I" %}
14652 ins_encode %{
14653 __ movdl($dst$$Register, $src$$XMMRegister);
14654 %}
14655 ins_pipe( pipe_slow );
14656 %}
14657
14658 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
14659 match(Set dst (MoveD2L src));
14660 effect(DEF dst, USE src);
14661 ins_cost(85);
14662 format %{ "movd $dst,$src\t# MoveD2L" %}
14663 ins_encode %{
14664 __ movdq($dst$$Register, $src$$XMMRegister);
14665 %}
14666 ins_pipe( pipe_slow );
14667 %}
14668
14669 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
14670 match(Set dst (MoveI2F src));
14671 effect(DEF dst, USE src);
14672 ins_cost(100);
14673 format %{ "movd $dst,$src\t# MoveI2F" %}
14674 ins_encode %{
14675 __ movdl($dst$$XMMRegister, $src$$Register);
14676 %}
14677 ins_pipe( pipe_slow );
14678 %}
14679
14680 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
14681 match(Set dst (MoveL2D src));
14682 effect(DEF dst, USE src);
14683 ins_cost(100);
14684 format %{ "movd $dst,$src\t# MoveL2D" %}
14685 ins_encode %{
14686 __ movdq($dst$$XMMRegister, $src$$Register);
14687 %}
14688 ins_pipe( pipe_slow );
14689 %}
14690
14691 // Fast clearing of an array
14692 // Small non-constant lenght ClearArray for non-AVX512 targets.
14693 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
14694 Universe dummy, rFlagsReg cr)
14695 %{
14696 predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
14697 match(Set dummy (ClearArray cnt base));
14698 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
14699
14700 format %{ $$template
14701 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14702 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14703 $$emit$$"jg LARGE\n\t"
14704 $$emit$$"dec rcx\n\t"
14705 $$emit$$"js DONE\t# Zero length\n\t"
14706 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14707 $$emit$$"dec rcx\n\t"
14708 $$emit$$"jge LOOP\n\t"
14709 $$emit$$"jmp DONE\n\t"
14710 $$emit$$"# LARGE:\n\t"
14711 if (UseFastStosb) {
14712 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14713 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14714 } else if (UseXMMForObjInit) {
14715 $$emit$$"mov rdi,rax\n\t"
14716 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14717 $$emit$$"jmpq L_zero_64_bytes\n\t"
14718 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14719 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14720 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14721 $$emit$$"add 0x40,rax\n\t"
14722 $$emit$$"# L_zero_64_bytes:\n\t"
14723 $$emit$$"sub 0x8,rcx\n\t"
14724 $$emit$$"jge L_loop\n\t"
14725 $$emit$$"add 0x4,rcx\n\t"
14726 $$emit$$"jl L_tail\n\t"
14727 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14728 $$emit$$"add 0x20,rax\n\t"
14729 $$emit$$"sub 0x4,rcx\n\t"
14730 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14731 $$emit$$"add 0x4,rcx\n\t"
14732 $$emit$$"jle L_end\n\t"
14733 $$emit$$"dec rcx\n\t"
14734 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14735 $$emit$$"vmovq xmm0,(rax)\n\t"
14736 $$emit$$"add 0x8,rax\n\t"
14737 $$emit$$"dec rcx\n\t"
14738 $$emit$$"jge L_sloop\n\t"
14739 $$emit$$"# L_end:\n\t"
14740 } else {
14741 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14742 }
14743 $$emit$$"# DONE"
14744 %}
14745 ins_encode %{
14746 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14747 $tmp$$XMMRegister, false, knoreg);
14748 %}
14749 ins_pipe(pipe_slow);
14750 %}
14751
14752 // Small non-constant length ClearArray for AVX512 targets.
14753 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
14754 Universe dummy, rFlagsReg cr)
14755 %{
14756 predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
14757 match(Set dummy (ClearArray cnt base));
14758 ins_cost(125);
14759 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
14760
14761 format %{ $$template
14762 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14763 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14764 $$emit$$"jg LARGE\n\t"
14765 $$emit$$"dec rcx\n\t"
14766 $$emit$$"js DONE\t# Zero length\n\t"
14767 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14768 $$emit$$"dec rcx\n\t"
14769 $$emit$$"jge LOOP\n\t"
14770 $$emit$$"jmp DONE\n\t"
14771 $$emit$$"# LARGE:\n\t"
14772 if (UseFastStosb) {
14773 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14774 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14775 } else if (UseXMMForObjInit) {
14776 $$emit$$"mov rdi,rax\n\t"
14777 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14778 $$emit$$"jmpq L_zero_64_bytes\n\t"
14779 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14780 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14781 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14782 $$emit$$"add 0x40,rax\n\t"
14783 $$emit$$"# L_zero_64_bytes:\n\t"
14784 $$emit$$"sub 0x8,rcx\n\t"
14785 $$emit$$"jge L_loop\n\t"
14786 $$emit$$"add 0x4,rcx\n\t"
14787 $$emit$$"jl L_tail\n\t"
14788 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14789 $$emit$$"add 0x20,rax\n\t"
14790 $$emit$$"sub 0x4,rcx\n\t"
14791 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14792 $$emit$$"add 0x4,rcx\n\t"
14793 $$emit$$"jle L_end\n\t"
14794 $$emit$$"dec rcx\n\t"
14795 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14796 $$emit$$"vmovq xmm0,(rax)\n\t"
14797 $$emit$$"add 0x8,rax\n\t"
14798 $$emit$$"dec rcx\n\t"
14799 $$emit$$"jge L_sloop\n\t"
14800 $$emit$$"# L_end:\n\t"
14801 } else {
14802 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14803 }
14804 $$emit$$"# DONE"
14805 %}
14806 ins_encode %{
14807 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14808 $tmp$$XMMRegister, false, $ktmp$$KRegister);
14809 %}
14810 ins_pipe(pipe_slow);
14811 %}
14812
14813 // Large non-constant length ClearArray for non-AVX512 targets.
14814 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
14815 Universe dummy, rFlagsReg cr)
14816 %{
14817 predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
14818 match(Set dummy (ClearArray cnt base));
14819 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
14820
14821 format %{ $$template
14822 if (UseFastStosb) {
14823 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14824 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14825 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
14826 } else if (UseXMMForObjInit) {
14827 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
14828 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14829 $$emit$$"jmpq L_zero_64_bytes\n\t"
14830 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14831 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14832 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14833 $$emit$$"add 0x40,rax\n\t"
14834 $$emit$$"# L_zero_64_bytes:\n\t"
14835 $$emit$$"sub 0x8,rcx\n\t"
14836 $$emit$$"jge L_loop\n\t"
14837 $$emit$$"add 0x4,rcx\n\t"
14838 $$emit$$"jl L_tail\n\t"
14839 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14840 $$emit$$"add 0x20,rax\n\t"
14841 $$emit$$"sub 0x4,rcx\n\t"
14842 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14843 $$emit$$"add 0x4,rcx\n\t"
14844 $$emit$$"jle L_end\n\t"
14845 $$emit$$"dec rcx\n\t"
14846 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14847 $$emit$$"vmovq xmm0,(rax)\n\t"
14848 $$emit$$"add 0x8,rax\n\t"
14849 $$emit$$"dec rcx\n\t"
14850 $$emit$$"jge L_sloop\n\t"
14851 $$emit$$"# L_end:\n\t"
14852 } else {
14853 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14854 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
14855 }
14856 %}
14857 ins_encode %{
14858 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14859 $tmp$$XMMRegister, true, knoreg);
14860 %}
14861 ins_pipe(pipe_slow);
14862 %}
14863
14864 // Large non-constant length ClearArray for AVX512 targets.
14865 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
14866 Universe dummy, rFlagsReg cr)
14867 %{
14868 predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
14869 match(Set dummy (ClearArray cnt base));
14870 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
14871
14872 format %{ $$template
14873 if (UseFastStosb) {
14874 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14875 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14876 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
14877 } else if (UseXMMForObjInit) {
14878 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
14879 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14880 $$emit$$"jmpq L_zero_64_bytes\n\t"
14881 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14882 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14883 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14884 $$emit$$"add 0x40,rax\n\t"
14885 $$emit$$"# L_zero_64_bytes:\n\t"
14886 $$emit$$"sub 0x8,rcx\n\t"
14887 $$emit$$"jge L_loop\n\t"
14888 $$emit$$"add 0x4,rcx\n\t"
14889 $$emit$$"jl L_tail\n\t"
14890 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14891 $$emit$$"add 0x20,rax\n\t"
14892 $$emit$$"sub 0x4,rcx\n\t"
14893 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14894 $$emit$$"add 0x4,rcx\n\t"
14895 $$emit$$"jle L_end\n\t"
14896 $$emit$$"dec rcx\n\t"
14897 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14898 $$emit$$"vmovq xmm0,(rax)\n\t"
14899 $$emit$$"add 0x8,rax\n\t"
14900 $$emit$$"dec rcx\n\t"
14901 $$emit$$"jge L_sloop\n\t"
14902 $$emit$$"# L_end:\n\t"
14903 } else {
14904 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14905 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
14906 }
14907 %}
14908 ins_encode %{
14909 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14910 $tmp$$XMMRegister, true, $ktmp$$KRegister);
14911 %}
14912 ins_pipe(pipe_slow);
14913 %}
14914
14915 // Small constant length ClearArray for AVX512 targets.
14916 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
14917 %{
14918 predicate(!((ClearArrayNode*)n)->is_large() && (MaxVectorSize >= 32) && VM_Version::supports_avx512vl());
14919 match(Set dummy (ClearArray cnt base));
14920 ins_cost(100);
14921 effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
14922 format %{ "clear_mem_imm $base , $cnt \n\t" %}
14923 ins_encode %{
14924 __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
14925 %}
14926 ins_pipe(pipe_slow);
14927 %}
14928
14929 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
14930 rax_RegI result, legRegD tmp1, rFlagsReg cr)
14931 %{
14932 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
14933 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14934 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14935
14936 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
14937 ins_encode %{
14938 __ string_compare($str1$$Register, $str2$$Register,
14939 $cnt1$$Register, $cnt2$$Register, $result$$Register,
14940 $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
14941 %}
14942 ins_pipe( pipe_slow );
14943 %}
14944
14945 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
14946 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
14947 %{
14948 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
14949 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14950 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14951
14952 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
14953 ins_encode %{
14954 __ string_compare($str1$$Register, $str2$$Register,
14955 $cnt1$$Register, $cnt2$$Register, $result$$Register,
14956 $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
14957 %}
14958 ins_pipe( pipe_slow );
14959 %}
14960
14961 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
14962 rax_RegI result, legRegD tmp1, rFlagsReg cr)
14963 %{
14964 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
14965 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14966 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14967
14968 format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
14969 ins_encode %{
14970 __ string_compare($str1$$Register, $str2$$Register,
14971 $cnt1$$Register, $cnt2$$Register, $result$$Register,
14972 $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
14973 %}
14974 ins_pipe( pipe_slow );
14975 %}
14976
14977 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
14978 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
14979 %{
14980 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
14981 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14982 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14983
14984 format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
14985 ins_encode %{
14986 __ string_compare($str1$$Register, $str2$$Register,
14987 $cnt1$$Register, $cnt2$$Register, $result$$Register,
14988 $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
14989 %}
14990 ins_pipe( pipe_slow );
14991 %}
14992
14993 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
14994 rax_RegI result, legRegD tmp1, rFlagsReg cr)
14995 %{
14996 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
14997 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14998 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14999
15000 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15001 ins_encode %{
15002 __ string_compare($str1$$Register, $str2$$Register,
15003 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15004 $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
15005 %}
15006 ins_pipe( pipe_slow );
15007 %}
15008
15009 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
15010 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
15011 %{
15012 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
15013 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15014 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15015
15016 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15017 ins_encode %{
15018 __ string_compare($str1$$Register, $str2$$Register,
15019 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15020 $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
15021 %}
15022 ins_pipe( pipe_slow );
15023 %}
15024
15025 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
15026 rax_RegI result, legRegD tmp1, rFlagsReg cr)
15027 %{
15028 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
15029 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15030 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15031
15032 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15033 ins_encode %{
15034 __ string_compare($str2$$Register, $str1$$Register,
15035 $cnt2$$Register, $cnt1$$Register, $result$$Register,
15036 $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
15037 %}
15038 ins_pipe( pipe_slow );
15039 %}
15040
15041 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
15042 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
15043 %{
15044 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
15045 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15046 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15047
15048 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15049 ins_encode %{
15050 __ string_compare($str2$$Register, $str1$$Register,
15051 $cnt2$$Register, $cnt1$$Register, $result$$Register,
15052 $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
15053 %}
15054 ins_pipe( pipe_slow );
15055 %}
15056
15057 // fast search of substring with known size.
15058 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
15059 rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
15060 %{
15061 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
15062 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15063 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
15064
15065 format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
15066 ins_encode %{
15067 int icnt2 = (int)$int_cnt2$$constant;
15068 if (icnt2 >= 16) {
15069 // IndexOf for constant substrings with size >= 16 elements
15070 // which don't need to be loaded through stack.
15071 __ string_indexofC8($str1$$Register, $str2$$Register,
15072 $cnt1$$Register, $cnt2$$Register,
15073 icnt2, $result$$Register,
15074 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
15075 } else {
15076 // Small strings are loaded through stack if they cross page boundary.
15077 __ string_indexof($str1$$Register, $str2$$Register,
15078 $cnt1$$Register, $cnt2$$Register,
15079 icnt2, $result$$Register,
15080 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
15081 }
15082 %}
15083 ins_pipe( pipe_slow );
15084 %}
15085
15086 // fast search of substring with known size.
15087 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
15088 rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
15089 %{
15090 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
15091 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15092 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
15093
15094 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
15095 ins_encode %{
15096 int icnt2 = (int)$int_cnt2$$constant;
15097 if (icnt2 >= 8) {
15098 // IndexOf for constant substrings with size >= 8 elements
15099 // which don't need to be loaded through stack.
15100 __ string_indexofC8($str1$$Register, $str2$$Register,
15101 $cnt1$$Register, $cnt2$$Register,
15102 icnt2, $result$$Register,
15103 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
15104 } else {
15105 // Small strings are loaded through stack if they cross page boundary.
15106 __ string_indexof($str1$$Register, $str2$$Register,
15107 $cnt1$$Register, $cnt2$$Register,
15108 icnt2, $result$$Register,
15109 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
15110 }
15111 %}
15112 ins_pipe( pipe_slow );
15113 %}
15114
15115 // fast search of substring with known size.
15116 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
15117 rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
15118 %{
15119 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
15120 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15121 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
15122
15123 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
15124 ins_encode %{
15125 int icnt2 = (int)$int_cnt2$$constant;
15126 if (icnt2 >= 8) {
15127 // IndexOf for constant substrings with size >= 8 elements
15128 // which don't need to be loaded through stack.
15129 __ string_indexofC8($str1$$Register, $str2$$Register,
15130 $cnt1$$Register, $cnt2$$Register,
15131 icnt2, $result$$Register,
15132 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
15133 } else {
15134 // Small strings are loaded through stack if they cross page boundary.
15135 __ string_indexof($str1$$Register, $str2$$Register,
15136 $cnt1$$Register, $cnt2$$Register,
15137 icnt2, $result$$Register,
15138 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
15139 }
15140 %}
15141 ins_pipe( pipe_slow );
15142 %}
15143
15144 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
15145 rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
15146 %{
15147 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
15148 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15149 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
15150
15151 format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
15152 ins_encode %{
15153 __ string_indexof($str1$$Register, $str2$$Register,
15154 $cnt1$$Register, $cnt2$$Register,
15155 (-1), $result$$Register,
15156 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
15157 %}
15158 ins_pipe( pipe_slow );
15159 %}
15160
15161 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
15162 rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
15163 %{
15164 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
15165 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15166 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
15167
15168 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
15169 ins_encode %{
15170 __ string_indexof($str1$$Register, $str2$$Register,
15171 $cnt1$$Register, $cnt2$$Register,
15172 (-1), $result$$Register,
15173 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
15174 %}
15175 ins_pipe( pipe_slow );
15176 %}
15177
15178 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
15179 rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
15180 %{
15181 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
15182 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15183 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
15184
15185 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
15186 ins_encode %{
15187 __ string_indexof($str1$$Register, $str2$$Register,
15188 $cnt1$$Register, $cnt2$$Register,
15189 (-1), $result$$Register,
15190 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
15191 %}
15192 ins_pipe( pipe_slow );
15193 %}
15194
15195 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
15196 rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
15197 %{
15198 predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15199 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15200 effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
15201 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result // KILL all" %}
15202 ins_encode %{
15203 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
15204 $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
15205 %}
15206 ins_pipe( pipe_slow );
15207 %}
15208
15209 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
15210 rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
15211 %{
15212 predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15213 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15214 effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
15215 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result // KILL all" %}
15216 ins_encode %{
15217 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
15218 $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
15219 %}
15220 ins_pipe( pipe_slow );
15221 %}
15222
15223 // fast string equals
15224 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
15225 legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
15226 %{
15227 predicate(!VM_Version::supports_avx512vlbw());
15228 match(Set result (StrEquals (Binary str1 str2) cnt));
15229 effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
15230
15231 format %{ "String Equals $str1,$str2,$cnt -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15232 ins_encode %{
15233 __ arrays_equals(false, $str1$$Register, $str2$$Register,
15234 $cnt$$Register, $result$$Register, $tmp3$$Register,
15235 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
15236 %}
15237 ins_pipe( pipe_slow );
15238 %}
15239
15240 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
15241 legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
15242 %{
15243 predicate(VM_Version::supports_avx512vlbw());
15244 match(Set result (StrEquals (Binary str1 str2) cnt));
15245 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
15246
15247 format %{ "String Equals $str1,$str2,$cnt -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15248 ins_encode %{
15249 __ arrays_equals(false, $str1$$Register, $str2$$Register,
15250 $cnt$$Register, $result$$Register, $tmp3$$Register,
15251 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
15252 %}
15253 ins_pipe( pipe_slow );
15254 %}
15255
15256 // fast array equals
15257 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15258 legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15259 %{
15260 predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
15261 match(Set result (AryEq ary1 ary2));
15262 effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15263
15264 format %{ "Array Equals byte[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15265 ins_encode %{
15266 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15267 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15268 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
15269 %}
15270 ins_pipe( pipe_slow );
15271 %}
15272
15273 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15274 legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15275 %{
15276 predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
15277 match(Set result (AryEq ary1 ary2));
15278 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15279
15280 format %{ "Array Equals byte[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15281 ins_encode %{
15282 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15283 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15284 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
15285 %}
15286 ins_pipe( pipe_slow );
15287 %}
15288
15289 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15290 legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15291 %{
15292 predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
15293 match(Set result (AryEq ary1 ary2));
15294 effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15295
15296 format %{ "Array Equals char[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15297 ins_encode %{
15298 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15299 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15300 $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
15301 %}
15302 ins_pipe( pipe_slow );
15303 %}
15304
15305 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15306 legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15307 %{
15308 predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
15309 match(Set result (AryEq ary1 ary2));
15310 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15311
15312 format %{ "Array Equals char[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15313 ins_encode %{
15314 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15315 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15316 $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
15317 %}
15318 ins_pipe( pipe_slow );
15319 %}
15320
15321 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
15322 legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
15323 legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
15324 legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
15325 legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
15326 %{
15327 predicate(UseAVX >= 2);
15328 match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
15329 effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
15330 TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
15331 TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
15332 USE basic_type, KILL cr);
15333
15334 format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result // KILL all" %}
15335 ins_encode %{
15336 __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
15337 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
15338 $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
15339 $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
15340 $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
15341 $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
15342 $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
15343 %}
15344 ins_pipe( pipe_slow );
15345 %}
15346
15347 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
15348 legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
15349 %{
15350 predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
15351 match(Set result (CountPositives ary1 len));
15352 effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
15353
15354 format %{ "countPositives byte[] $ary1,$len -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15355 ins_encode %{
15356 __ count_positives($ary1$$Register, $len$$Register,
15357 $result$$Register, $tmp3$$Register,
15358 $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
15359 %}
15360 ins_pipe( pipe_slow );
15361 %}
15362
15363 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
15364 legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
15365 %{
15366 predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
15367 match(Set result (CountPositives ary1 len));
15368 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
15369
15370 format %{ "countPositives byte[] $ary1,$len -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15371 ins_encode %{
15372 __ count_positives($ary1$$Register, $len$$Register,
15373 $result$$Register, $tmp3$$Register,
15374 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
15375 %}
15376 ins_pipe( pipe_slow );
15377 %}
15378
15379 // fast char[] to byte[] compression
15380 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
15381 legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15382 predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
15383 match(Set result (StrCompressedCopy src (Binary dst len)));
15384 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
15385 USE_KILL len, KILL tmp5, KILL cr);
15386
15387 format %{ "String Compress $src,$dst -> $result // KILL RAX, RCX, RDX" %}
15388 ins_encode %{
15389 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
15390 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15391 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
15392 knoreg, knoreg);
15393 %}
15394 ins_pipe( pipe_slow );
15395 %}
15396
15397 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
15398 legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15399 predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
15400 match(Set result (StrCompressedCopy src (Binary dst len)));
15401 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
15402 USE_KILL len, KILL tmp5, KILL cr);
15403
15404 format %{ "String Compress $src,$dst -> $result // KILL RAX, RCX, RDX" %}
15405 ins_encode %{
15406 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
15407 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15408 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
15409 $ktmp1$$KRegister, $ktmp2$$KRegister);
15410 %}
15411 ins_pipe( pipe_slow );
15412 %}
15413 // fast byte[] to char[] inflation
15414 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15415 legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
15416 predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
15417 match(Set dummy (StrInflatedCopy src (Binary dst len)));
15418 effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
15419
15420 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
15421 ins_encode %{
15422 __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
15423 $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
15424 %}
15425 ins_pipe( pipe_slow );
15426 %}
15427
15428 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15429 legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
15430 predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
15431 match(Set dummy (StrInflatedCopy src (Binary dst len)));
15432 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
15433
15434 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
15435 ins_encode %{
15436 __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
15437 $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
15438 %}
15439 ins_pipe( pipe_slow );
15440 %}
15441
15442 // encode char[] to byte[] in ISO_8859_1
15443 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15444 legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
15445 rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15446 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
15447 match(Set result (EncodeISOArray src (Binary dst len)));
15448 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
15449
15450 format %{ "Encode iso array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
15451 ins_encode %{
15452 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
15453 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15454 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
15455 %}
15456 ins_pipe( pipe_slow );
15457 %}
15458
15459 // encode char[] to byte[] in ASCII
15460 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15461 legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
15462 rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15463 predicate(((EncodeISOArrayNode*)n)->is_ascii());
15464 match(Set result (EncodeISOArray src (Binary dst len)));
15465 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
15466
15467 format %{ "Encode ascii array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
15468 ins_encode %{
15469 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
15470 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15471 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
15472 %}
15473 ins_pipe( pipe_slow );
15474 %}
15475
15476 //----------Overflow Math Instructions-----------------------------------------
15477
15478 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
15479 %{
15480 match(Set cr (OverflowAddI op1 op2));
15481 effect(DEF cr, USE_KILL op1, USE op2);
15482
15483 format %{ "addl $op1, $op2\t# overflow check int" %}
15484
15485 ins_encode %{
15486 __ addl($op1$$Register, $op2$$Register);
15487 %}
15488 ins_pipe(ialu_reg_reg);
15489 %}
15490
15491 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
15492 %{
15493 match(Set cr (OverflowAddI op1 op2));
15494 effect(DEF cr, USE_KILL op1, USE op2);
15495
15496 format %{ "addl $op1, $op2\t# overflow check int" %}
15497
15498 ins_encode %{
15499 __ addl($op1$$Register, $op2$$constant);
15500 %}
15501 ins_pipe(ialu_reg_reg);
15502 %}
15503
15504 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
15505 %{
15506 match(Set cr (OverflowAddL op1 op2));
15507 effect(DEF cr, USE_KILL op1, USE op2);
15508
15509 format %{ "addq $op1, $op2\t# overflow check long" %}
15510 ins_encode %{
15511 __ addq($op1$$Register, $op2$$Register);
15512 %}
15513 ins_pipe(ialu_reg_reg);
15514 %}
15515
15516 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
15517 %{
15518 match(Set cr (OverflowAddL op1 op2));
15519 effect(DEF cr, USE_KILL op1, USE op2);
15520
15521 format %{ "addq $op1, $op2\t# overflow check long" %}
15522 ins_encode %{
15523 __ addq($op1$$Register, $op2$$constant);
15524 %}
15525 ins_pipe(ialu_reg_reg);
15526 %}
15527
15528 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
15529 %{
15530 match(Set cr (OverflowSubI op1 op2));
15531
15532 format %{ "cmpl $op1, $op2\t# overflow check int" %}
15533 ins_encode %{
15534 __ cmpl($op1$$Register, $op2$$Register);
15535 %}
15536 ins_pipe(ialu_reg_reg);
15537 %}
15538
15539 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
15540 %{
15541 match(Set cr (OverflowSubI op1 op2));
15542
15543 format %{ "cmpl $op1, $op2\t# overflow check int" %}
15544 ins_encode %{
15545 __ cmpl($op1$$Register, $op2$$constant);
15546 %}
15547 ins_pipe(ialu_reg_reg);
15548 %}
15549
15550 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
15551 %{
15552 match(Set cr (OverflowSubL op1 op2));
15553
15554 format %{ "cmpq $op1, $op2\t# overflow check long" %}
15555 ins_encode %{
15556 __ cmpq($op1$$Register, $op2$$Register);
15557 %}
15558 ins_pipe(ialu_reg_reg);
15559 %}
15560
15561 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
15562 %{
15563 match(Set cr (OverflowSubL op1 op2));
15564
15565 format %{ "cmpq $op1, $op2\t# overflow check long" %}
15566 ins_encode %{
15567 __ cmpq($op1$$Register, $op2$$constant);
15568 %}
15569 ins_pipe(ialu_reg_reg);
15570 %}
15571
15572 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
15573 %{
15574 match(Set cr (OverflowSubI zero op2));
15575 effect(DEF cr, USE_KILL op2);
15576
15577 format %{ "negl $op2\t# overflow check int" %}
15578 ins_encode %{
15579 __ negl($op2$$Register);
15580 %}
15581 ins_pipe(ialu_reg_reg);
15582 %}
15583
15584 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
15585 %{
15586 match(Set cr (OverflowSubL zero op2));
15587 effect(DEF cr, USE_KILL op2);
15588
15589 format %{ "negq $op2\t# overflow check long" %}
15590 ins_encode %{
15591 __ negq($op2$$Register);
15592 %}
15593 ins_pipe(ialu_reg_reg);
15594 %}
15595
15596 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
15597 %{
15598 match(Set cr (OverflowMulI op1 op2));
15599 effect(DEF cr, USE_KILL op1, USE op2);
15600
15601 format %{ "imull $op1, $op2\t# overflow check int" %}
15602 ins_encode %{
15603 __ imull($op1$$Register, $op2$$Register);
15604 %}
15605 ins_pipe(ialu_reg_reg_alu0);
15606 %}
15607
15608 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
15609 %{
15610 match(Set cr (OverflowMulI op1 op2));
15611 effect(DEF cr, TEMP tmp, USE op1, USE op2);
15612
15613 format %{ "imull $tmp, $op1, $op2\t# overflow check int" %}
15614 ins_encode %{
15615 __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
15616 %}
15617 ins_pipe(ialu_reg_reg_alu0);
15618 %}
15619
15620 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
15621 %{
15622 match(Set cr (OverflowMulL op1 op2));
15623 effect(DEF cr, USE_KILL op1, USE op2);
15624
15625 format %{ "imulq $op1, $op2\t# overflow check long" %}
15626 ins_encode %{
15627 __ imulq($op1$$Register, $op2$$Register);
15628 %}
15629 ins_pipe(ialu_reg_reg_alu0);
15630 %}
15631
15632 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
15633 %{
15634 match(Set cr (OverflowMulL op1 op2));
15635 effect(DEF cr, TEMP tmp, USE op1, USE op2);
15636
15637 format %{ "imulq $tmp, $op1, $op2\t# overflow check long" %}
15638 ins_encode %{
15639 __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
15640 %}
15641 ins_pipe(ialu_reg_reg_alu0);
15642 %}
15643
15644
15645 //----------Control Flow Instructions------------------------------------------
15646 // Signed compare Instructions
15647
15648 // XXX more variants!!
15649 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
15650 %{
15651 match(Set cr (CmpI op1 op2));
15652 effect(DEF cr, USE op1, USE op2);
15653
15654 format %{ "cmpl $op1, $op2" %}
15655 ins_encode %{
15656 __ cmpl($op1$$Register, $op2$$Register);
15657 %}
15658 ins_pipe(ialu_cr_reg_reg);
15659 %}
15660
15661 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
15662 %{
15663 match(Set cr (CmpI op1 op2));
15664
15665 format %{ "cmpl $op1, $op2" %}
15666 ins_encode %{
15667 __ cmpl($op1$$Register, $op2$$constant);
15668 %}
15669 ins_pipe(ialu_cr_reg_imm);
15670 %}
15671
15672 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
15673 %{
15674 match(Set cr (CmpI op1 (LoadI op2)));
15675
15676 ins_cost(500); // XXX
15677 format %{ "cmpl $op1, $op2" %}
15678 ins_encode %{
15679 __ cmpl($op1$$Register, $op2$$Address);
15680 %}
15681 ins_pipe(ialu_cr_reg_mem);
15682 %}
15683
15684 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
15685 %{
15686 match(Set cr (CmpI src zero));
15687
15688 format %{ "testl $src, $src" %}
15689 ins_encode %{
15690 __ testl($src$$Register, $src$$Register);
15691 %}
15692 ins_pipe(ialu_cr_reg_imm);
15693 %}
15694
15695 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
15696 %{
15697 match(Set cr (CmpI (AndI src con) zero));
15698
15699 format %{ "testl $src, $con" %}
15700 ins_encode %{
15701 __ testl($src$$Register, $con$$constant);
15702 %}
15703 ins_pipe(ialu_cr_reg_imm);
15704 %}
15705
15706 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
15707 %{
15708 match(Set cr (CmpI (AndI src1 src2) zero));
15709
15710 format %{ "testl $src1, $src2" %}
15711 ins_encode %{
15712 __ testl($src1$$Register, $src2$$Register);
15713 %}
15714 ins_pipe(ialu_cr_reg_imm);
15715 %}
15716
15717 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
15718 %{
15719 match(Set cr (CmpI (AndI src (LoadI mem)) zero));
15720
15721 format %{ "testl $src, $mem" %}
15722 ins_encode %{
15723 __ testl($src$$Register, $mem$$Address);
15724 %}
15725 ins_pipe(ialu_cr_reg_mem);
15726 %}
15727
15728 // Unsigned compare Instructions; really, same as signed except they
15729 // produce an rFlagsRegU instead of rFlagsReg.
15730 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
15731 %{
15732 match(Set cr (CmpU op1 op2));
15733
15734 format %{ "cmpl $op1, $op2\t# unsigned" %}
15735 ins_encode %{
15736 __ cmpl($op1$$Register, $op2$$Register);
15737 %}
15738 ins_pipe(ialu_cr_reg_reg);
15739 %}
15740
15741 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
15742 %{
15743 match(Set cr (CmpU op1 op2));
15744
15745 format %{ "cmpl $op1, $op2\t# unsigned" %}
15746 ins_encode %{
15747 __ cmpl($op1$$Register, $op2$$constant);
15748 %}
15749 ins_pipe(ialu_cr_reg_imm);
15750 %}
15751
15752 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
15753 %{
15754 match(Set cr (CmpU op1 (LoadI op2)));
15755
15756 ins_cost(500); // XXX
15757 format %{ "cmpl $op1, $op2\t# unsigned" %}
15758 ins_encode %{
15759 __ cmpl($op1$$Register, $op2$$Address);
15760 %}
15761 ins_pipe(ialu_cr_reg_mem);
15762 %}
15763
15764 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
15765 %{
15766 match(Set cr (CmpU src zero));
15767
15768 format %{ "testl $src, $src\t# unsigned" %}
15769 ins_encode %{
15770 __ testl($src$$Register, $src$$Register);
15771 %}
15772 ins_pipe(ialu_cr_reg_imm);
15773 %}
15774
15775 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
15776 %{
15777 match(Set cr (CmpP op1 op2));
15778
15779 format %{ "cmpq $op1, $op2\t# ptr" %}
15780 ins_encode %{
15781 __ cmpq($op1$$Register, $op2$$Register);
15782 %}
15783 ins_pipe(ialu_cr_reg_reg);
15784 %}
15785
15786 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
15787 %{
15788 match(Set cr (CmpP op1 (LoadP op2)));
15789 predicate(n->in(2)->as_Load()->barrier_data() == 0);
15790
15791 ins_cost(500); // XXX
15792 format %{ "cmpq $op1, $op2\t# ptr" %}
15793 ins_encode %{
15794 __ cmpq($op1$$Register, $op2$$Address);
15795 %}
15796 ins_pipe(ialu_cr_reg_mem);
15797 %}
15798
15799 // XXX this is generalized by compP_rReg_mem???
15800 // Compare raw pointer (used in out-of-heap check).
15801 // Only works because non-oop pointers must be raw pointers
15802 // and raw pointers have no anti-dependencies.
15803 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
15804 %{
15805 predicate(n->in(2)->in(2)->bottom_type()->isa_rawptr() != nullptr &&
15806 n->in(2)->as_Load()->barrier_data() == 0);
15807 match(Set cr (CmpP op1 (LoadP op2)));
15808
15809 format %{ "cmpq $op1, $op2\t# raw ptr" %}
15810 ins_encode %{
15811 __ cmpq($op1$$Register, $op2$$Address);
15812 %}
15813 ins_pipe(ialu_cr_reg_mem);
15814 %}
15815
15816 // This will generate a signed flags result. This should be OK since
15817 // any compare to a zero should be eq/neq.
15818 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
15819 %{
15820 match(Set cr (CmpP src zero));
15821
15822 format %{ "testq $src, $src\t# ptr" %}
15823 ins_encode %{
15824 __ testq($src$$Register, $src$$Register);
15825 %}
15826 ins_pipe(ialu_cr_reg_imm);
15827 %}
15828
15829 // This will generate a signed flags result. This should be OK since
15830 // any compare to a zero should be eq/neq.
15831 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
15832 %{
15833 predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
15834 n->in(1)->as_Load()->barrier_data() == 0);
15835 match(Set cr (CmpP (LoadP op) zero));
15836
15837 ins_cost(500); // XXX
15838 format %{ "testq $op, 0xffffffffffffffff\t# ptr" %}
15839 ins_encode %{
15840 __ testq($op$$Address, 0xFFFFFFFF);
15841 %}
15842 ins_pipe(ialu_cr_reg_imm);
15843 %}
15844
15845 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
15846 %{
15847 predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
15848 n->in(1)->as_Load()->barrier_data() == 0);
15849 match(Set cr (CmpP (LoadP mem) zero));
15850
15851 format %{ "cmpq R12, $mem\t# ptr (R12_heapbase==0)" %}
15852 ins_encode %{
15853 __ cmpq(r12, $mem$$Address);
15854 %}
15855 ins_pipe(ialu_cr_reg_mem);
15856 %}
15857
15858 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
15859 %{
15860 match(Set cr (CmpN op1 op2));
15861
15862 format %{ "cmpl $op1, $op2\t# compressed ptr" %}
15863 ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
15864 ins_pipe(ialu_cr_reg_reg);
15865 %}
15866
15867 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
15868 %{
15869 predicate(n->in(2)->as_Load()->barrier_data() == 0);
15870 match(Set cr (CmpN src (LoadN mem)));
15871
15872 format %{ "cmpl $src, $mem\t# compressed ptr" %}
15873 ins_encode %{
15874 __ cmpl($src$$Register, $mem$$Address);
15875 %}
15876 ins_pipe(ialu_cr_reg_mem);
15877 %}
15878
15879 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
15880 match(Set cr (CmpN op1 op2));
15881
15882 format %{ "cmpl $op1, $op2\t# compressed ptr" %}
15883 ins_encode %{
15884 __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
15885 %}
15886 ins_pipe(ialu_cr_reg_imm);
15887 %}
15888
15889 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
15890 %{
15891 predicate(n->in(2)->as_Load()->barrier_data() == 0);
15892 match(Set cr (CmpN src (LoadN mem)));
15893
15894 format %{ "cmpl $mem, $src\t# compressed ptr" %}
15895 ins_encode %{
15896 __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
15897 %}
15898 ins_pipe(ialu_cr_reg_mem);
15899 %}
15900
15901 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
15902 match(Set cr (CmpN op1 op2));
15903
15904 format %{ "cmpl $op1, $op2\t# compressed klass ptr" %}
15905 ins_encode %{
15906 __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
15907 %}
15908 ins_pipe(ialu_cr_reg_imm);
15909 %}
15910
15911 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
15912 %{
15913 predicate(!UseCompactObjectHeaders);
15914 match(Set cr (CmpN src (LoadNKlass mem)));
15915
15916 format %{ "cmpl $mem, $src\t# compressed klass ptr" %}
15917 ins_encode %{
15918 __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
15919 %}
15920 ins_pipe(ialu_cr_reg_mem);
15921 %}
15922
15923 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
15924 match(Set cr (CmpN src zero));
15925
15926 format %{ "testl $src, $src\t# compressed ptr" %}
15927 ins_encode %{ __ testl($src$$Register, $src$$Register); %}
15928 ins_pipe(ialu_cr_reg_imm);
15929 %}
15930
15931 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
15932 %{
15933 predicate(CompressedOops::base() != nullptr &&
15934 n->in(1)->as_Load()->barrier_data() == 0);
15935 match(Set cr (CmpN (LoadN mem) zero));
15936
15937 ins_cost(500); // XXX
15938 format %{ "testl $mem, 0xffffffff\t# compressed ptr" %}
15939 ins_encode %{
15940 __ cmpl($mem$$Address, (int)0xFFFFFFFF);
15941 %}
15942 ins_pipe(ialu_cr_reg_mem);
15943 %}
15944
15945 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
15946 %{
15947 predicate(CompressedOops::base() == nullptr &&
15948 n->in(1)->as_Load()->barrier_data() == 0);
15949 match(Set cr (CmpN (LoadN mem) zero));
15950
15951 format %{ "cmpl R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
15952 ins_encode %{
15953 __ cmpl(r12, $mem$$Address);
15954 %}
15955 ins_pipe(ialu_cr_reg_mem);
15956 %}
15957
15958 // Yanked all unsigned pointer compare operations.
15959 // Pointer compares are done with CmpP which is already unsigned.
15960
15961 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
15962 %{
15963 match(Set cr (CmpL op1 op2));
15964
15965 format %{ "cmpq $op1, $op2" %}
15966 ins_encode %{
15967 __ cmpq($op1$$Register, $op2$$Register);
15968 %}
15969 ins_pipe(ialu_cr_reg_reg);
15970 %}
15971
15972 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
15973 %{
15974 match(Set cr (CmpL op1 op2));
15975
15976 format %{ "cmpq $op1, $op2" %}
15977 ins_encode %{
15978 __ cmpq($op1$$Register, $op2$$constant);
15979 %}
15980 ins_pipe(ialu_cr_reg_imm);
15981 %}
15982
15983 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
15984 %{
15985 match(Set cr (CmpL op1 (LoadL op2)));
15986
15987 format %{ "cmpq $op1, $op2" %}
15988 ins_encode %{
15989 __ cmpq($op1$$Register, $op2$$Address);
15990 %}
15991 ins_pipe(ialu_cr_reg_mem);
15992 %}
15993
15994 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
15995 %{
15996 match(Set cr (CmpL src zero));
15997
15998 format %{ "testq $src, $src" %}
15999 ins_encode %{
16000 __ testq($src$$Register, $src$$Register);
16001 %}
16002 ins_pipe(ialu_cr_reg_imm);
16003 %}
16004
16005 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
16006 %{
16007 match(Set cr (CmpL (AndL src con) zero));
16008
16009 format %{ "testq $src, $con\t# long" %}
16010 ins_encode %{
16011 __ testq($src$$Register, $con$$constant);
16012 %}
16013 ins_pipe(ialu_cr_reg_imm);
16014 %}
16015
16016 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
16017 %{
16018 match(Set cr (CmpL (AndL src1 src2) zero));
16019
16020 format %{ "testq $src1, $src2\t# long" %}
16021 ins_encode %{
16022 __ testq($src1$$Register, $src2$$Register);
16023 %}
16024 ins_pipe(ialu_cr_reg_imm);
16025 %}
16026
16027 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
16028 %{
16029 match(Set cr (CmpL (AndL src (LoadL mem)) zero));
16030
16031 format %{ "testq $src, $mem" %}
16032 ins_encode %{
16033 __ testq($src$$Register, $mem$$Address);
16034 %}
16035 ins_pipe(ialu_cr_reg_mem);
16036 %}
16037
16038 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
16039 %{
16040 match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
16041
16042 format %{ "testq $src, $mem" %}
16043 ins_encode %{
16044 __ testq($src$$Register, $mem$$Address);
16045 %}
16046 ins_pipe(ialu_cr_reg_mem);
16047 %}
16048
16049 // Manifest a CmpU result in an integer register. Very painful.
16050 // This is the test to avoid.
16051 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
16052 %{
16053 match(Set dst (CmpU3 src1 src2));
16054 effect(KILL flags);
16055
16056 ins_cost(275); // XXX
16057 format %{ "cmpl $src1, $src2\t# CmpL3\n\t"
16058 "movl $dst, -1\n\t"
16059 "jb,u done\n\t"
16060 "setcc $dst \t# emits setne + movzbl or setzune for APX"
16061 "done:" %}
16062 ins_encode %{
16063 Label done;
16064 __ cmpl($src1$$Register, $src2$$Register);
16065 __ movl($dst$$Register, -1);
16066 __ jccb(Assembler::below, done);
16067 __ setcc(Assembler::notZero, $dst$$Register);
16068 __ bind(done);
16069 %}
16070 ins_pipe(pipe_slow);
16071 %}
16072
16073 // Manifest a CmpL result in an integer register. Very painful.
16074 // This is the test to avoid.
16075 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
16076 %{
16077 match(Set dst (CmpL3 src1 src2));
16078 effect(KILL flags);
16079
16080 ins_cost(275); // XXX
16081 format %{ "cmpq $src1, $src2\t# CmpL3\n\t"
16082 "movl $dst, -1\n\t"
16083 "jl,s done\n\t"
16084 "setcc $dst \t# emits setne + movzbl or setzune for APX"
16085 "done:" %}
16086 ins_encode %{
16087 Label done;
16088 __ cmpq($src1$$Register, $src2$$Register);
16089 __ movl($dst$$Register, -1);
16090 __ jccb(Assembler::less, done);
16091 __ setcc(Assembler::notZero, $dst$$Register);
16092 __ bind(done);
16093 %}
16094 ins_pipe(pipe_slow);
16095 %}
16096
16097 // Manifest a CmpUL result in an integer register. Very painful.
16098 // This is the test to avoid.
16099 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
16100 %{
16101 match(Set dst (CmpUL3 src1 src2));
16102 effect(KILL flags);
16103
16104 ins_cost(275); // XXX
16105 format %{ "cmpq $src1, $src2\t# CmpL3\n\t"
16106 "movl $dst, -1\n\t"
16107 "jb,u done\n\t"
16108 "setcc $dst \t# emits setne + movzbl or setzune for APX"
16109 "done:" %}
16110 ins_encode %{
16111 Label done;
16112 __ cmpq($src1$$Register, $src2$$Register);
16113 __ movl($dst$$Register, -1);
16114 __ jccb(Assembler::below, done);
16115 __ setcc(Assembler::notZero, $dst$$Register);
16116 __ bind(done);
16117 %}
16118 ins_pipe(pipe_slow);
16119 %}
16120
16121 // Unsigned long compare Instructions; really, same as signed long except they
16122 // produce an rFlagsRegU instead of rFlagsReg.
16123 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
16124 %{
16125 match(Set cr (CmpUL op1 op2));
16126
16127 format %{ "cmpq $op1, $op2\t# unsigned" %}
16128 ins_encode %{
16129 __ cmpq($op1$$Register, $op2$$Register);
16130 %}
16131 ins_pipe(ialu_cr_reg_reg);
16132 %}
16133
16134 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
16135 %{
16136 match(Set cr (CmpUL op1 op2));
16137
16138 format %{ "cmpq $op1, $op2\t# unsigned" %}
16139 ins_encode %{
16140 __ cmpq($op1$$Register, $op2$$constant);
16141 %}
16142 ins_pipe(ialu_cr_reg_imm);
16143 %}
16144
16145 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
16146 %{
16147 match(Set cr (CmpUL op1 (LoadL op2)));
16148
16149 format %{ "cmpq $op1, $op2\t# unsigned" %}
16150 ins_encode %{
16151 __ cmpq($op1$$Register, $op2$$Address);
16152 %}
16153 ins_pipe(ialu_cr_reg_mem);
16154 %}
16155
16156 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
16157 %{
16158 match(Set cr (CmpUL src zero));
16159
16160 format %{ "testq $src, $src\t# unsigned" %}
16161 ins_encode %{
16162 __ testq($src$$Register, $src$$Register);
16163 %}
16164 ins_pipe(ialu_cr_reg_imm);
16165 %}
16166
16167 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
16168 %{
16169 match(Set cr (CmpI (LoadB mem) imm));
16170
16171 ins_cost(125);
16172 format %{ "cmpb $mem, $imm" %}
16173 ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
16174 ins_pipe(ialu_cr_reg_mem);
16175 %}
16176
16177 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
16178 %{
16179 match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
16180
16181 ins_cost(125);
16182 format %{ "testb $mem, $imm\t# ubyte" %}
16183 ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
16184 ins_pipe(ialu_cr_reg_mem);
16185 %}
16186
16187 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
16188 %{
16189 match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
16190
16191 ins_cost(125);
16192 format %{ "testb $mem, $imm\t# byte" %}
16193 ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
16194 ins_pipe(ialu_cr_reg_mem);
16195 %}
16196
16197 //----------Max and Min--------------------------------------------------------
16198 // Min Instructions
16199
16200 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
16201 %{
16202 predicate(!UseAPX);
16203 effect(USE_DEF dst, USE src, USE cr);
16204
16205 format %{ "cmovlgt $dst, $src\t# min" %}
16206 ins_encode %{
16207 __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
16208 %}
16209 ins_pipe(pipe_cmov_reg);
16210 %}
16211
16212 instruct cmovI_reg_g_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
16213 %{
16214 predicate(UseAPX);
16215 effect(DEF dst, USE src1, USE src2, USE cr);
16216
16217 format %{ "ecmovlgt $dst, $src1, $src2\t# min ndd" %}
16218 ins_encode %{
16219 __ ecmovl(Assembler::greater, $dst$$Register, $src1$$Register, $src2$$Register);
16220 %}
16221 ins_pipe(pipe_cmov_reg);
16222 %}
16223
16224 instruct minI_rReg(rRegI dst, rRegI src)
16225 %{
16226 predicate(!UseAPX);
16227 match(Set dst (MinI dst src));
16228
16229 ins_cost(200);
16230 expand %{
16231 rFlagsReg cr;
16232 compI_rReg(cr, dst, src);
16233 cmovI_reg_g(dst, src, cr);
16234 %}
16235 %}
16236
16237 instruct minI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
16238 %{
16239 predicate(UseAPX);
16240 match(Set dst (MinI src1 src2));
16241 effect(DEF dst, USE src1, USE src2);
16242 flag(PD::Flag_ndd_demotable_opr1);
16243
16244 ins_cost(200);
16245 expand %{
16246 rFlagsReg cr;
16247 compI_rReg(cr, src1, src2);
16248 cmovI_reg_g_ndd(dst, src1, src2, cr);
16249 %}
16250 %}
16251
16252 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
16253 %{
16254 predicate(!UseAPX);
16255 effect(USE_DEF dst, USE src, USE cr);
16256
16257 format %{ "cmovllt $dst, $src\t# max" %}
16258 ins_encode %{
16259 __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
16260 %}
16261 ins_pipe(pipe_cmov_reg);
16262 %}
16263
16264 instruct cmovI_reg_l_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
16265 %{
16266 predicate(UseAPX);
16267 effect(DEF dst, USE src1, USE src2, USE cr);
16268
16269 format %{ "ecmovllt $dst, $src1, $src2\t# max ndd" %}
16270 ins_encode %{
16271 __ ecmovl(Assembler::less, $dst$$Register, $src1$$Register, $src2$$Register);
16272 %}
16273 ins_pipe(pipe_cmov_reg);
16274 %}
16275
16276 instruct maxI_rReg(rRegI dst, rRegI src)
16277 %{
16278 predicate(!UseAPX);
16279 match(Set dst (MaxI dst src));
16280
16281 ins_cost(200);
16282 expand %{
16283 rFlagsReg cr;
16284 compI_rReg(cr, dst, src);
16285 cmovI_reg_l(dst, src, cr);
16286 %}
16287 %}
16288
16289 instruct maxI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
16290 %{
16291 predicate(UseAPX);
16292 match(Set dst (MaxI src1 src2));
16293 effect(DEF dst, USE src1, USE src2);
16294 flag(PD::Flag_ndd_demotable_opr1);
16295
16296 ins_cost(200);
16297 expand %{
16298 rFlagsReg cr;
16299 compI_rReg(cr, src1, src2);
16300 cmovI_reg_l_ndd(dst, src1, src2, cr);
16301 %}
16302 %}
16303
16304 // ============================================================================
16305 // Branch Instructions
16306
16307 // Jump Direct - Label defines a relative address from JMP+1
16308 instruct jmpDir(label labl)
16309 %{
16310 match(Goto);
16311 effect(USE labl);
16312
16313 ins_cost(300);
16314 format %{ "jmp $labl" %}
16315 size(5);
16316 ins_encode %{
16317 Label* L = $labl$$label;
16318 __ jmp(*L, false); // Always long jump
16319 %}
16320 ins_pipe(pipe_jmp);
16321 %}
16322
16323 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16324 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
16325 %{
16326 match(If cop cr);
16327 effect(USE labl);
16328
16329 ins_cost(300);
16330 format %{ "j$cop $labl" %}
16331 size(6);
16332 ins_encode %{
16333 Label* L = $labl$$label;
16334 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16335 %}
16336 ins_pipe(pipe_jcc);
16337 %}
16338
16339 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16340 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
16341 %{
16342 match(CountedLoopEnd cop cr);
16343 effect(USE labl);
16344
16345 ins_cost(300);
16346 format %{ "j$cop $labl\t# loop end" %}
16347 size(6);
16348 ins_encode %{
16349 Label* L = $labl$$label;
16350 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16351 %}
16352 ins_pipe(pipe_jcc);
16353 %}
16354
16355 // Jump Direct Conditional - using unsigned comparison
16356 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
16357 match(If cop cmp);
16358 effect(USE labl);
16359
16360 ins_cost(300);
16361 format %{ "j$cop,u $labl" %}
16362 size(6);
16363 ins_encode %{
16364 Label* L = $labl$$label;
16365 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16366 %}
16367 ins_pipe(pipe_jcc);
16368 %}
16369
16370 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
16371 match(If cop cmp);
16372 effect(USE labl);
16373
16374 ins_cost(200);
16375 format %{ "j$cop,u $labl" %}
16376 size(6);
16377 ins_encode %{
16378 Label* L = $labl$$label;
16379 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16380 %}
16381 ins_pipe(pipe_jcc);
16382 %}
16383
16384 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
16385 match(If cop cmp);
16386 effect(USE labl);
16387
16388 ins_cost(200);
16389 format %{ $$template
16390 if ($cop$$cmpcode == Assembler::notEqual) {
16391 $$emit$$"jp,u $labl\n\t"
16392 $$emit$$"j$cop,u $labl"
16393 } else {
16394 $$emit$$"jp,u done\n\t"
16395 $$emit$$"j$cop,u $labl\n\t"
16396 $$emit$$"done:"
16397 }
16398 %}
16399 ins_encode %{
16400 Label* l = $labl$$label;
16401 if ($cop$$cmpcode == Assembler::notEqual) {
16402 __ jcc(Assembler::parity, *l, false);
16403 __ jcc(Assembler::notEqual, *l, false);
16404 } else if ($cop$$cmpcode == Assembler::equal) {
16405 Label done;
16406 __ jccb(Assembler::parity, done);
16407 __ jcc(Assembler::equal, *l, false);
16408 __ bind(done);
16409 } else {
16410 ShouldNotReachHere();
16411 }
16412 %}
16413 ins_pipe(pipe_jcc);
16414 %}
16415
16416 // Jump Direct Conditional - using signed and unsigned comparison
16417 instruct jmpConUCFE(cmpOpUCFE cop, rFlagsRegUCFE cmp, label labl) %{
16418 match(If cop cmp);
16419 effect(USE labl);
16420
16421 ins_cost(200);
16422 format %{ "j$cop,su $labl" %}
16423 size(6);
16424 ins_encode %{
16425 Label* L = $labl$$label;
16426 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16427 %}
16428 ins_pipe(pipe_jcc);
16429 %}
16430
16431 // ============================================================================
16432 // The 2nd slow-half of a subtype check. Scan the subklass's 2ndary
16433 // superklass array for an instance of the superklass. Set a hidden
16434 // internal cache on a hit (cache is checked with exposed code in
16435 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16436 // encoding ALSO sets flags.
16437
16438 instruct partialSubtypeCheck(rdi_RegP result,
16439 rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
16440 rFlagsReg cr)
16441 %{
16442 match(Set result (PartialSubtypeCheck sub super));
16443 predicate(!UseSecondarySupersTable);
16444 effect(KILL rcx, KILL cr);
16445
16446 ins_cost(1100); // slightly larger than the next version
16447 format %{ "movq rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
16448 "movl rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
16449 "addq rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
16450 "repne scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
16451 "jne,s miss\t\t# Missed: rdi not-zero\n\t"
16452 "movq [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
16453 "xorq $result, $result\t\t Hit: rdi zero\n\t"
16454 "miss:\t" %}
16455
16456 ins_encode %{
16457 Label miss;
16458 // NB: Callers may assume that, when $result is a valid register,
16459 // check_klass_subtype_slow_path_linear sets it to a nonzero
16460 // value.
16461 __ check_klass_subtype_slow_path_linear($sub$$Register, $super$$Register,
16462 $rcx$$Register, $result$$Register,
16463 nullptr, &miss,
16464 /*set_cond_codes:*/ true);
16465 __ xorptr($result$$Register, $result$$Register);
16466 __ bind(miss);
16467 %}
16468
16469 ins_pipe(pipe_slow);
16470 %}
16471
16472 // ============================================================================
16473 // Two versions of hashtable-based partialSubtypeCheck, both used when
16474 // we need to search for a super class in the secondary supers array.
16475 // The first is used when we don't know _a priori_ the class being
16476 // searched for. The second, far more common, is used when we do know:
16477 // this is used for instanceof, checkcast, and any case where C2 can
16478 // determine it by constant propagation.
16479
16480 instruct partialSubtypeCheckVarSuper(rsi_RegP sub, rax_RegP super, rdi_RegP result,
16481 rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
16482 rFlagsReg cr)
16483 %{
16484 match(Set result (PartialSubtypeCheck sub super));
16485 predicate(UseSecondarySupersTable);
16486 effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
16487
16488 ins_cost(1000);
16489 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16490
16491 ins_encode %{
16492 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register, $temp1$$Register, $temp2$$Register,
16493 $temp3$$Register, $temp4$$Register, $result$$Register);
16494 %}
16495
16496 ins_pipe(pipe_slow);
16497 %}
16498
16499 instruct partialSubtypeCheckConstSuper(rsi_RegP sub, rax_RegP super_reg, immP super_con, rdi_RegP result,
16500 rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
16501 rFlagsReg cr)
16502 %{
16503 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16504 predicate(UseSecondarySupersTable);
16505 effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
16506
16507 ins_cost(700); // smaller than the next version
16508 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16509
16510 ins_encode %{
16511 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16512 if (InlineSecondarySupersTest) {
16513 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register, $temp1$$Register, $temp2$$Register,
16514 $temp3$$Register, $temp4$$Register, $result$$Register,
16515 super_klass_slot);
16516 } else {
16517 __ call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16518 }
16519 %}
16520
16521 ins_pipe(pipe_slow);
16522 %}
16523
16524 // ============================================================================
16525 // Branch Instructions -- short offset versions
16526 //
16527 // These instructions are used to replace jumps of a long offset (the default
16528 // match) with jumps of a shorter offset. These instructions are all tagged
16529 // with the ins_short_branch attribute, which causes the ADLC to suppress the
16530 // match rules in general matching. Instead, the ADLC generates a conversion
16531 // method in the MachNode which can be used to do in-place replacement of the
16532 // long variant with the shorter variant. The compiler will determine if a
16533 // branch can be taken by the is_short_branch_offset() predicate in the machine
16534 // specific code section of the file.
16535
16536 // Jump Direct - Label defines a relative address from JMP+1
16537 instruct jmpDir_short(label labl) %{
16538 match(Goto);
16539 effect(USE labl);
16540
16541 ins_cost(300);
16542 format %{ "jmp,s $labl" %}
16543 size(2);
16544 ins_encode %{
16545 Label* L = $labl$$label;
16546 __ jmpb(*L);
16547 %}
16548 ins_pipe(pipe_jmp);
16549 ins_short_branch(1);
16550 %}
16551
16552 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16553 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
16554 match(If cop cr);
16555 effect(USE labl);
16556
16557 ins_cost(300);
16558 format %{ "j$cop,s $labl" %}
16559 size(2);
16560 ins_encode %{
16561 Label* L = $labl$$label;
16562 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16563 %}
16564 ins_pipe(pipe_jcc);
16565 ins_short_branch(1);
16566 %}
16567
16568 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16569 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
16570 match(CountedLoopEnd cop cr);
16571 effect(USE labl);
16572
16573 ins_cost(300);
16574 format %{ "j$cop,s $labl\t# loop end" %}
16575 size(2);
16576 ins_encode %{
16577 Label* L = $labl$$label;
16578 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16579 %}
16580 ins_pipe(pipe_jcc);
16581 ins_short_branch(1);
16582 %}
16583
16584 // Jump Direct Conditional - using unsigned comparison
16585 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
16586 match(If cop cmp);
16587 effect(USE labl);
16588
16589 ins_cost(300);
16590 format %{ "j$cop,us $labl" %}
16591 size(2);
16592 ins_encode %{
16593 Label* L = $labl$$label;
16594 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16595 %}
16596 ins_pipe(pipe_jcc);
16597 ins_short_branch(1);
16598 %}
16599
16600 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
16601 match(If cop cmp);
16602 effect(USE labl);
16603
16604 ins_cost(300);
16605 format %{ "j$cop,us $labl" %}
16606 size(2);
16607 ins_encode %{
16608 Label* L = $labl$$label;
16609 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16610 %}
16611 ins_pipe(pipe_jcc);
16612 ins_short_branch(1);
16613 %}
16614
16615 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
16616 match(If cop cmp);
16617 effect(USE labl);
16618
16619 ins_cost(300);
16620 format %{ $$template
16621 if ($cop$$cmpcode == Assembler::notEqual) {
16622 $$emit$$"jp,u,s $labl\n\t"
16623 $$emit$$"j$cop,u,s $labl"
16624 } else {
16625 $$emit$$"jp,u,s done\n\t"
16626 $$emit$$"j$cop,u,s $labl\n\t"
16627 $$emit$$"done:"
16628 }
16629 %}
16630 size(4);
16631 ins_encode %{
16632 Label* l = $labl$$label;
16633 if ($cop$$cmpcode == Assembler::notEqual) {
16634 __ jccb(Assembler::parity, *l);
16635 __ jccb(Assembler::notEqual, *l);
16636 } else if ($cop$$cmpcode == Assembler::equal) {
16637 Label done;
16638 __ jccb(Assembler::parity, done);
16639 __ jccb(Assembler::equal, *l);
16640 __ bind(done);
16641 } else {
16642 ShouldNotReachHere();
16643 }
16644 %}
16645 ins_pipe(pipe_jcc);
16646 ins_short_branch(1);
16647 %}
16648
16649 // Jump Direct Conditional - using signed and unsigned comparison
16650 instruct jmpConUCFE_short(cmpOpUCFE cop, rFlagsRegUCFE cmp, label labl) %{
16651 match(If cop cmp);
16652 effect(USE labl);
16653
16654 ins_cost(300);
16655 format %{ "j$cop,sus $labl" %}
16656 size(2);
16657 ins_encode %{
16658 Label* L = $labl$$label;
16659 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16660 %}
16661 ins_pipe(pipe_jcc);
16662 ins_short_branch(1);
16663 %}
16664
16665 // ============================================================================
16666 // inlined locking and unlocking
16667
16668 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
16669 match(Set cr (FastLock object box));
16670 effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
16671 ins_cost(300);
16672 format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
16673 ins_encode %{
16674 __ fast_lock($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
16675 %}
16676 ins_pipe(pipe_slow);
16677 %}
16678
16679 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
16680 match(Set cr (FastUnlock object rax_reg));
16681 effect(TEMP tmp, USE_KILL rax_reg);
16682 ins_cost(300);
16683 format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
16684 ins_encode %{
16685 __ fast_unlock($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
16686 %}
16687 ins_pipe(pipe_slow);
16688 %}
16689
16690
16691 // ============================================================================
16692 // Safepoint Instructions
16693 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
16694 %{
16695 match(SafePoint poll);
16696 effect(KILL cr, USE poll);
16697
16698 format %{ "testl rax, [$poll]\t"
16699 "# Safepoint: poll for GC" %}
16700 ins_cost(125);
16701 ins_encode %{
16702 __ relocate(relocInfo::poll_type);
16703 address pre_pc = __ pc();
16704 __ testl(rax, Address($poll$$Register, 0));
16705 assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
16706 %}
16707 ins_pipe(ialu_reg_mem);
16708 %}
16709
16710 instruct mask_all_evexL(kReg dst, rRegL src) %{
16711 match(Set dst (MaskAll src));
16712 format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
16713 ins_encode %{
16714 int mask_len = Matcher::vector_length(this);
16715 __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
16716 %}
16717 ins_pipe( pipe_slow );
16718 %}
16719
16720 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
16721 predicate(Matcher::vector_length(n) > 32);
16722 match(Set dst (MaskAll src));
16723 effect(TEMP tmp);
16724 format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
16725 ins_encode %{
16726 int mask_len = Matcher::vector_length(this);
16727 __ movslq($tmp$$Register, $src$$Register);
16728 __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
16729 %}
16730 ins_pipe( pipe_slow );
16731 %}
16732
16733 // ============================================================================
16734 // Procedure Call/Return Instructions
16735 // Call Java Static Instruction
16736 // Note: If this code changes, the corresponding ret_addr_offset() and
16737 // compute_padding() functions will have to be adjusted.
16738 instruct CallStaticJavaDirect(method meth) %{
16739 match(CallStaticJava);
16740 effect(USE meth);
16741
16742 ins_cost(300);
16743 format %{ "call,static " %}
16744 opcode(0xE8); /* E8 cd */
16745 ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
16746 ins_pipe(pipe_slow);
16747 ins_alignment(4);
16748 %}
16749
16750 // Call Java Dynamic Instruction
16751 // Note: If this code changes, the corresponding ret_addr_offset() and
16752 // compute_padding() functions will have to be adjusted.
16753 instruct CallDynamicJavaDirect(method meth)
16754 %{
16755 match(CallDynamicJava);
16756 effect(USE meth);
16757
16758 ins_cost(300);
16759 format %{ "movq rax, #Universe::non_oop_word()\n\t"
16760 "call,dynamic " %}
16761 ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
16762 ins_pipe(pipe_slow);
16763 ins_alignment(4);
16764 %}
16765
16766 // Call Runtime Instruction
16767 instruct CallRuntimeDirect(method meth)
16768 %{
16769 match(CallRuntime);
16770 effect(USE meth);
16771
16772 ins_cost(300);
16773 format %{ "call,runtime " %}
16774 ins_encode(clear_avx, Java_To_Runtime(meth));
16775 ins_pipe(pipe_slow);
16776 %}
16777
16778 // Call runtime without safepoint
16779 instruct CallLeafDirect(method meth)
16780 %{
16781 match(CallLeaf);
16782 effect(USE meth);
16783
16784 ins_cost(300);
16785 format %{ "call_leaf,runtime " %}
16786 ins_encode(clear_avx, Java_To_Runtime(meth));
16787 ins_pipe(pipe_slow);
16788 %}
16789
16790 // Call runtime without safepoint and with vector arguments
16791 instruct CallLeafDirectVector(method meth)
16792 %{
16793 match(CallLeafVector);
16794 effect(USE meth);
16795
16796 ins_cost(300);
16797 format %{ "call_leaf,vector " %}
16798 ins_encode(Java_To_Runtime(meth));
16799 ins_pipe(pipe_slow);
16800 %}
16801
16802 // Call runtime without safepoint
16803 instruct CallLeafNoFPDirect(method meth)
16804 %{
16805 match(CallLeafNoFP);
16806 effect(USE meth);
16807
16808 ins_cost(300);
16809 format %{ "call_leaf_nofp,runtime " %}
16810 ins_encode(clear_avx, Java_To_Runtime(meth));
16811 ins_pipe(pipe_slow);
16812 %}
16813
16814 // Return Instruction
16815 // Remove the return address & jump to it.
16816 // Notice: We always emit a nop after a ret to make sure there is room
16817 // for safepoint patching
16818 instruct Ret()
16819 %{
16820 match(Return);
16821
16822 format %{ "ret" %}
16823 ins_encode %{
16824 __ ret(0);
16825 %}
16826 ins_pipe(pipe_jmp);
16827 %}
16828
16829 // Tail Call; Jump from runtime stub to Java code.
16830 // Also known as an 'interprocedural jump'.
16831 // Target of jump will eventually return to caller.
16832 // TailJump below removes the return address.
16833 // Don't use rbp for 'jump_target' because a MachEpilogNode has already been
16834 // emitted just above the TailCall which has reset rbp to the caller state.
16835 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
16836 %{
16837 match(TailCall jump_target method_ptr);
16838
16839 ins_cost(300);
16840 format %{ "jmp $jump_target\t# rbx holds method" %}
16841 ins_encode %{
16842 __ jmp($jump_target$$Register);
16843 %}
16844 ins_pipe(pipe_jmp);
16845 %}
16846
16847 // Tail Jump; remove the return address; jump to target.
16848 // TailCall above leaves the return address around.
16849 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
16850 %{
16851 match(TailJump jump_target ex_oop);
16852
16853 ins_cost(300);
16854 format %{ "popq rdx\t# pop return address\n\t"
16855 "jmp $jump_target" %}
16856 ins_encode %{
16857 __ popq(as_Register(RDX_enc));
16858 __ jmp($jump_target$$Register);
16859 %}
16860 ins_pipe(pipe_jmp);
16861 %}
16862
16863 // Forward exception.
16864 instruct ForwardExceptionjmp()
16865 %{
16866 match(ForwardException);
16867
16868 format %{ "jmp forward_exception_stub" %}
16869 ins_encode %{
16870 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()), noreg);
16871 %}
16872 ins_pipe(pipe_jmp);
16873 %}
16874
16875 // Create exception oop: created by stack-crawling runtime code.
16876 // Created exception is now available to this handler, and is setup
16877 // just prior to jumping to this handler. No code emitted.
16878 instruct CreateException(rax_RegP ex_oop)
16879 %{
16880 match(Set ex_oop (CreateEx));
16881
16882 size(0);
16883 // use the following format syntax
16884 format %{ "# exception oop is in rax; no code emitted" %}
16885 ins_encode();
16886 ins_pipe(empty);
16887 %}
16888
16889 // Rethrow exception:
16890 // The exception oop will come in the first argument position.
16891 // Then JUMP (not call) to the rethrow stub code.
16892 instruct RethrowException()
16893 %{
16894 match(Rethrow);
16895
16896 // use the following format syntax
16897 format %{ "jmp rethrow_stub" %}
16898 ins_encode %{
16899 __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
16900 %}
16901 ins_pipe(pipe_jmp);
16902 %}
16903
16904 // ============================================================================
16905 // This name is KNOWN by the ADLC and cannot be changed.
16906 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16907 // for this guy.
16908 instruct tlsLoadP(r15_RegP dst) %{
16909 match(Set dst (ThreadLocal));
16910 effect(DEF dst);
16911
16912 size(0);
16913 format %{ "# TLS is in R15" %}
16914 ins_encode( /*empty encoding*/ );
16915 ins_pipe(ialu_reg_reg);
16916 %}
16917
16918 instruct addF_reg(regF dst, regF src) %{
16919 predicate(UseAVX == 0);
16920 match(Set dst (AddF dst src));
16921
16922 format %{ "addss $dst, $src" %}
16923 ins_cost(150);
16924 ins_encode %{
16925 __ addss($dst$$XMMRegister, $src$$XMMRegister);
16926 %}
16927 ins_pipe(pipe_slow);
16928 %}
16929
16930 instruct addF_mem(regF dst, memory src) %{
16931 predicate(UseAVX == 0);
16932 match(Set dst (AddF dst (LoadF src)));
16933
16934 format %{ "addss $dst, $src" %}
16935 ins_cost(150);
16936 ins_encode %{
16937 __ addss($dst$$XMMRegister, $src$$Address);
16938 %}
16939 ins_pipe(pipe_slow);
16940 %}
16941
16942 instruct addF_imm(regF dst, immF con) %{
16943 predicate(UseAVX == 0);
16944 match(Set dst (AddF dst con));
16945 format %{ "addss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
16946 ins_cost(150);
16947 ins_encode %{
16948 __ addss($dst$$XMMRegister, $constantaddress($con));
16949 %}
16950 ins_pipe(pipe_slow);
16951 %}
16952
16953 instruct addF_reg_reg(regF dst, regF src1, regF src2) %{
16954 predicate(UseAVX > 0);
16955 match(Set dst (AddF src1 src2));
16956
16957 format %{ "vaddss $dst, $src1, $src2" %}
16958 ins_cost(150);
16959 ins_encode %{
16960 __ vaddss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
16961 %}
16962 ins_pipe(pipe_slow);
16963 %}
16964
16965 instruct addF_reg_mem(regF dst, regF src1, memory src2) %{
16966 predicate(UseAVX > 0);
16967 match(Set dst (AddF src1 (LoadF src2)));
16968
16969 format %{ "vaddss $dst, $src1, $src2" %}
16970 ins_cost(150);
16971 ins_encode %{
16972 __ vaddss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
16973 %}
16974 ins_pipe(pipe_slow);
16975 %}
16976
16977 instruct addF_reg_imm(regF dst, regF src, immF con) %{
16978 predicate(UseAVX > 0);
16979 match(Set dst (AddF src con));
16980
16981 format %{ "vaddss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
16982 ins_cost(150);
16983 ins_encode %{
16984 __ vaddss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
16985 %}
16986 ins_pipe(pipe_slow);
16987 %}
16988
16989 instruct addD_reg(regD dst, regD src) %{
16990 predicate(UseAVX == 0);
16991 match(Set dst (AddD dst src));
16992
16993 format %{ "addsd $dst, $src" %}
16994 ins_cost(150);
16995 ins_encode %{
16996 __ addsd($dst$$XMMRegister, $src$$XMMRegister);
16997 %}
16998 ins_pipe(pipe_slow);
16999 %}
17000
17001 instruct addD_mem(regD dst, memory src) %{
17002 predicate(UseAVX == 0);
17003 match(Set dst (AddD dst (LoadD src)));
17004
17005 format %{ "addsd $dst, $src" %}
17006 ins_cost(150);
17007 ins_encode %{
17008 __ addsd($dst$$XMMRegister, $src$$Address);
17009 %}
17010 ins_pipe(pipe_slow);
17011 %}
17012
17013 instruct addD_imm(regD dst, immD con) %{
17014 predicate(UseAVX == 0);
17015 match(Set dst (AddD dst con));
17016 format %{ "addsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17017 ins_cost(150);
17018 ins_encode %{
17019 __ addsd($dst$$XMMRegister, $constantaddress($con));
17020 %}
17021 ins_pipe(pipe_slow);
17022 %}
17023
17024 instruct addD_reg_reg(regD dst, regD src1, regD src2) %{
17025 predicate(UseAVX > 0);
17026 match(Set dst (AddD src1 src2));
17027
17028 format %{ "vaddsd $dst, $src1, $src2" %}
17029 ins_cost(150);
17030 ins_encode %{
17031 __ vaddsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17032 %}
17033 ins_pipe(pipe_slow);
17034 %}
17035
17036 instruct addD_reg_mem(regD dst, regD src1, memory src2) %{
17037 predicate(UseAVX > 0);
17038 match(Set dst (AddD src1 (LoadD src2)));
17039
17040 format %{ "vaddsd $dst, $src1, $src2" %}
17041 ins_cost(150);
17042 ins_encode %{
17043 __ vaddsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17044 %}
17045 ins_pipe(pipe_slow);
17046 %}
17047
17048 instruct addD_reg_imm(regD dst, regD src, immD con) %{
17049 predicate(UseAVX > 0);
17050 match(Set dst (AddD src con));
17051
17052 format %{ "vaddsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17053 ins_cost(150);
17054 ins_encode %{
17055 __ vaddsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17056 %}
17057 ins_pipe(pipe_slow);
17058 %}
17059
17060 instruct subF_reg(regF dst, regF src) %{
17061 predicate(UseAVX == 0);
17062 match(Set dst (SubF dst src));
17063
17064 format %{ "subss $dst, $src" %}
17065 ins_cost(150);
17066 ins_encode %{
17067 __ subss($dst$$XMMRegister, $src$$XMMRegister);
17068 %}
17069 ins_pipe(pipe_slow);
17070 %}
17071
17072 instruct subF_mem(regF dst, memory src) %{
17073 predicate(UseAVX == 0);
17074 match(Set dst (SubF dst (LoadF src)));
17075
17076 format %{ "subss $dst, $src" %}
17077 ins_cost(150);
17078 ins_encode %{
17079 __ subss($dst$$XMMRegister, $src$$Address);
17080 %}
17081 ins_pipe(pipe_slow);
17082 %}
17083
17084 instruct subF_imm(regF dst, immF con) %{
17085 predicate(UseAVX == 0);
17086 match(Set dst (SubF dst con));
17087 format %{ "subss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
17088 ins_cost(150);
17089 ins_encode %{
17090 __ subss($dst$$XMMRegister, $constantaddress($con));
17091 %}
17092 ins_pipe(pipe_slow);
17093 %}
17094
17095 instruct subF_reg_reg(regF dst, regF src1, regF src2) %{
17096 predicate(UseAVX > 0);
17097 match(Set dst (SubF src1 src2));
17098
17099 format %{ "vsubss $dst, $src1, $src2" %}
17100 ins_cost(150);
17101 ins_encode %{
17102 __ vsubss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17103 %}
17104 ins_pipe(pipe_slow);
17105 %}
17106
17107 instruct subF_reg_mem(regF dst, regF src1, memory src2) %{
17108 predicate(UseAVX > 0);
17109 match(Set dst (SubF src1 (LoadF src2)));
17110
17111 format %{ "vsubss $dst, $src1, $src2" %}
17112 ins_cost(150);
17113 ins_encode %{
17114 __ vsubss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17115 %}
17116 ins_pipe(pipe_slow);
17117 %}
17118
17119 instruct subF_reg_imm(regF dst, regF src, immF con) %{
17120 predicate(UseAVX > 0);
17121 match(Set dst (SubF src con));
17122
17123 format %{ "vsubss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
17124 ins_cost(150);
17125 ins_encode %{
17126 __ vsubss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17127 %}
17128 ins_pipe(pipe_slow);
17129 %}
17130
17131 instruct subD_reg(regD dst, regD src) %{
17132 predicate(UseAVX == 0);
17133 match(Set dst (SubD dst src));
17134
17135 format %{ "subsd $dst, $src" %}
17136 ins_cost(150);
17137 ins_encode %{
17138 __ subsd($dst$$XMMRegister, $src$$XMMRegister);
17139 %}
17140 ins_pipe(pipe_slow);
17141 %}
17142
17143 instruct subD_mem(regD dst, memory src) %{
17144 predicate(UseAVX == 0);
17145 match(Set dst (SubD dst (LoadD src)));
17146
17147 format %{ "subsd $dst, $src" %}
17148 ins_cost(150);
17149 ins_encode %{
17150 __ subsd($dst$$XMMRegister, $src$$Address);
17151 %}
17152 ins_pipe(pipe_slow);
17153 %}
17154
17155 instruct subD_imm(regD dst, immD con) %{
17156 predicate(UseAVX == 0);
17157 match(Set dst (SubD dst con));
17158 format %{ "subsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17159 ins_cost(150);
17160 ins_encode %{
17161 __ subsd($dst$$XMMRegister, $constantaddress($con));
17162 %}
17163 ins_pipe(pipe_slow);
17164 %}
17165
17166 instruct subD_reg_reg(regD dst, regD src1, regD src2) %{
17167 predicate(UseAVX > 0);
17168 match(Set dst (SubD src1 src2));
17169
17170 format %{ "vsubsd $dst, $src1, $src2" %}
17171 ins_cost(150);
17172 ins_encode %{
17173 __ vsubsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17174 %}
17175 ins_pipe(pipe_slow);
17176 %}
17177
17178 instruct subD_reg_mem(regD dst, regD src1, memory src2) %{
17179 predicate(UseAVX > 0);
17180 match(Set dst (SubD src1 (LoadD src2)));
17181
17182 format %{ "vsubsd $dst, $src1, $src2" %}
17183 ins_cost(150);
17184 ins_encode %{
17185 __ vsubsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17186 %}
17187 ins_pipe(pipe_slow);
17188 %}
17189
17190 instruct subD_reg_imm(regD dst, regD src, immD con) %{
17191 predicate(UseAVX > 0);
17192 match(Set dst (SubD src con));
17193
17194 format %{ "vsubsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17195 ins_cost(150);
17196 ins_encode %{
17197 __ vsubsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17198 %}
17199 ins_pipe(pipe_slow);
17200 %}
17201
17202 instruct mulF_reg(regF dst, regF src) %{
17203 predicate(UseAVX == 0);
17204 match(Set dst (MulF dst src));
17205
17206 format %{ "mulss $dst, $src" %}
17207 ins_cost(150);
17208 ins_encode %{
17209 __ mulss($dst$$XMMRegister, $src$$XMMRegister);
17210 %}
17211 ins_pipe(pipe_slow);
17212 %}
17213
17214 instruct mulF_mem(regF dst, memory src) %{
17215 predicate(UseAVX == 0);
17216 match(Set dst (MulF dst (LoadF src)));
17217
17218 format %{ "mulss $dst, $src" %}
17219 ins_cost(150);
17220 ins_encode %{
17221 __ mulss($dst$$XMMRegister, $src$$Address);
17222 %}
17223 ins_pipe(pipe_slow);
17224 %}
17225
17226 instruct mulF_imm(regF dst, immF con) %{
17227 predicate(UseAVX == 0);
17228 match(Set dst (MulF dst con));
17229 format %{ "mulss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
17230 ins_cost(150);
17231 ins_encode %{
17232 __ mulss($dst$$XMMRegister, $constantaddress($con));
17233 %}
17234 ins_pipe(pipe_slow);
17235 %}
17236
17237 instruct mulF_reg_reg(regF dst, regF src1, regF src2) %{
17238 predicate(UseAVX > 0);
17239 match(Set dst (MulF src1 src2));
17240
17241 format %{ "vmulss $dst, $src1, $src2" %}
17242 ins_cost(150);
17243 ins_encode %{
17244 __ vmulss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17245 %}
17246 ins_pipe(pipe_slow);
17247 %}
17248
17249 instruct mulF_reg_mem(regF dst, regF src1, memory src2) %{
17250 predicate(UseAVX > 0);
17251 match(Set dst (MulF src1 (LoadF src2)));
17252
17253 format %{ "vmulss $dst, $src1, $src2" %}
17254 ins_cost(150);
17255 ins_encode %{
17256 __ vmulss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17257 %}
17258 ins_pipe(pipe_slow);
17259 %}
17260
17261 instruct mulF_reg_imm(regF dst, regF src, immF con) %{
17262 predicate(UseAVX > 0);
17263 match(Set dst (MulF src con));
17264
17265 format %{ "vmulss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
17266 ins_cost(150);
17267 ins_encode %{
17268 __ vmulss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17269 %}
17270 ins_pipe(pipe_slow);
17271 %}
17272
17273 instruct mulD_reg(regD dst, regD src) %{
17274 predicate(UseAVX == 0);
17275 match(Set dst (MulD dst src));
17276
17277 format %{ "mulsd $dst, $src" %}
17278 ins_cost(150);
17279 ins_encode %{
17280 __ mulsd($dst$$XMMRegister, $src$$XMMRegister);
17281 %}
17282 ins_pipe(pipe_slow);
17283 %}
17284
17285 instruct mulD_mem(regD dst, memory src) %{
17286 predicate(UseAVX == 0);
17287 match(Set dst (MulD dst (LoadD src)));
17288
17289 format %{ "mulsd $dst, $src" %}
17290 ins_cost(150);
17291 ins_encode %{
17292 __ mulsd($dst$$XMMRegister, $src$$Address);
17293 %}
17294 ins_pipe(pipe_slow);
17295 %}
17296
17297 instruct mulD_imm(regD dst, immD con) %{
17298 predicate(UseAVX == 0);
17299 match(Set dst (MulD dst con));
17300 format %{ "mulsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17301 ins_cost(150);
17302 ins_encode %{
17303 __ mulsd($dst$$XMMRegister, $constantaddress($con));
17304 %}
17305 ins_pipe(pipe_slow);
17306 %}
17307
17308 instruct mulD_reg_reg(regD dst, regD src1, regD src2) %{
17309 predicate(UseAVX > 0);
17310 match(Set dst (MulD src1 src2));
17311
17312 format %{ "vmulsd $dst, $src1, $src2" %}
17313 ins_cost(150);
17314 ins_encode %{
17315 __ vmulsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17316 %}
17317 ins_pipe(pipe_slow);
17318 %}
17319
17320 instruct mulD_reg_mem(regD dst, regD src1, memory src2) %{
17321 predicate(UseAVX > 0);
17322 match(Set dst (MulD src1 (LoadD src2)));
17323
17324 format %{ "vmulsd $dst, $src1, $src2" %}
17325 ins_cost(150);
17326 ins_encode %{
17327 __ vmulsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17328 %}
17329 ins_pipe(pipe_slow);
17330 %}
17331
17332 instruct mulD_reg_imm(regD dst, regD src, immD con) %{
17333 predicate(UseAVX > 0);
17334 match(Set dst (MulD src con));
17335
17336 format %{ "vmulsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17337 ins_cost(150);
17338 ins_encode %{
17339 __ vmulsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17340 %}
17341 ins_pipe(pipe_slow);
17342 %}
17343
17344 instruct divF_reg(regF dst, regF src) %{
17345 predicate(UseAVX == 0);
17346 match(Set dst (DivF dst src));
17347
17348 format %{ "divss $dst, $src" %}
17349 ins_cost(150);
17350 ins_encode %{
17351 __ divss($dst$$XMMRegister, $src$$XMMRegister);
17352 %}
17353 ins_pipe(pipe_slow);
17354 %}
17355
17356 instruct divF_mem(regF dst, memory src) %{
17357 predicate(UseAVX == 0);
17358 match(Set dst (DivF dst (LoadF src)));
17359
17360 format %{ "divss $dst, $src" %}
17361 ins_cost(150);
17362 ins_encode %{
17363 __ divss($dst$$XMMRegister, $src$$Address);
17364 %}
17365 ins_pipe(pipe_slow);
17366 %}
17367
17368 instruct divF_imm(regF dst, immF con) %{
17369 predicate(UseAVX == 0);
17370 match(Set dst (DivF dst con));
17371 format %{ "divss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
17372 ins_cost(150);
17373 ins_encode %{
17374 __ divss($dst$$XMMRegister, $constantaddress($con));
17375 %}
17376 ins_pipe(pipe_slow);
17377 %}
17378
17379 instruct divF_reg_reg(regF dst, regF src1, regF src2) %{
17380 predicate(UseAVX > 0);
17381 match(Set dst (DivF src1 src2));
17382
17383 format %{ "vdivss $dst, $src1, $src2" %}
17384 ins_cost(150);
17385 ins_encode %{
17386 __ vdivss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17387 %}
17388 ins_pipe(pipe_slow);
17389 %}
17390
17391 instruct divF_reg_mem(regF dst, regF src1, memory src2) %{
17392 predicate(UseAVX > 0);
17393 match(Set dst (DivF src1 (LoadF src2)));
17394
17395 format %{ "vdivss $dst, $src1, $src2" %}
17396 ins_cost(150);
17397 ins_encode %{
17398 __ vdivss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17399 %}
17400 ins_pipe(pipe_slow);
17401 %}
17402
17403 instruct divF_reg_imm(regF dst, regF src, immF con) %{
17404 predicate(UseAVX > 0);
17405 match(Set dst (DivF src con));
17406
17407 format %{ "vdivss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
17408 ins_cost(150);
17409 ins_encode %{
17410 __ vdivss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17411 %}
17412 ins_pipe(pipe_slow);
17413 %}
17414
17415 instruct divD_reg(regD dst, regD src) %{
17416 predicate(UseAVX == 0);
17417 match(Set dst (DivD dst src));
17418
17419 format %{ "divsd $dst, $src" %}
17420 ins_cost(150);
17421 ins_encode %{
17422 __ divsd($dst$$XMMRegister, $src$$XMMRegister);
17423 %}
17424 ins_pipe(pipe_slow);
17425 %}
17426
17427 instruct divD_mem(regD dst, memory src) %{
17428 predicate(UseAVX == 0);
17429 match(Set dst (DivD dst (LoadD src)));
17430
17431 format %{ "divsd $dst, $src" %}
17432 ins_cost(150);
17433 ins_encode %{
17434 __ divsd($dst$$XMMRegister, $src$$Address);
17435 %}
17436 ins_pipe(pipe_slow);
17437 %}
17438
17439 instruct divD_imm(regD dst, immD con) %{
17440 predicate(UseAVX == 0);
17441 match(Set dst (DivD dst con));
17442 format %{ "divsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17443 ins_cost(150);
17444 ins_encode %{
17445 __ divsd($dst$$XMMRegister, $constantaddress($con));
17446 %}
17447 ins_pipe(pipe_slow);
17448 %}
17449
17450 instruct divD_reg_reg(regD dst, regD src1, regD src2) %{
17451 predicate(UseAVX > 0);
17452 match(Set dst (DivD src1 src2));
17453
17454 format %{ "vdivsd $dst, $src1, $src2" %}
17455 ins_cost(150);
17456 ins_encode %{
17457 __ vdivsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17458 %}
17459 ins_pipe(pipe_slow);
17460 %}
17461
17462 instruct divD_reg_mem(regD dst, regD src1, memory src2) %{
17463 predicate(UseAVX > 0);
17464 match(Set dst (DivD src1 (LoadD src2)));
17465
17466 format %{ "vdivsd $dst, $src1, $src2" %}
17467 ins_cost(150);
17468 ins_encode %{
17469 __ vdivsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17470 %}
17471 ins_pipe(pipe_slow);
17472 %}
17473
17474 instruct divD_reg_imm(regD dst, regD src, immD con) %{
17475 predicate(UseAVX > 0);
17476 match(Set dst (DivD src con));
17477
17478 format %{ "vdivsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17479 ins_cost(150);
17480 ins_encode %{
17481 __ vdivsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17482 %}
17483 ins_pipe(pipe_slow);
17484 %}
17485
17486 instruct absF_reg(regF dst) %{
17487 predicate(UseAVX == 0);
17488 match(Set dst (AbsF dst));
17489 ins_cost(150);
17490 format %{ "andps $dst, [0x7fffffff]\t# abs float by sign masking" %}
17491 ins_encode %{
17492 __ andps($dst$$XMMRegister, ExternalAddress(float_signmask()));
17493 %}
17494 ins_pipe(pipe_slow);
17495 %}
17496
17497 instruct absF_reg_reg(vlRegF dst, vlRegF src) %{
17498 predicate(UseAVX > 0);
17499 match(Set dst (AbsF src));
17500 ins_cost(150);
17501 format %{ "vandps $dst, $src, [0x7fffffff]\t# abs float by sign masking" %}
17502 ins_encode %{
17503 int vlen_enc = Assembler::AVX_128bit;
17504 __ vandps($dst$$XMMRegister, $src$$XMMRegister,
17505 ExternalAddress(float_signmask()), vlen_enc);
17506 %}
17507 ins_pipe(pipe_slow);
17508 %}
17509
17510 instruct absD_reg(regD dst) %{
17511 predicate(UseAVX == 0);
17512 match(Set dst (AbsD dst));
17513 ins_cost(150);
17514 format %{ "andpd $dst, [0x7fffffffffffffff]\t"
17515 "# abs double by sign masking" %}
17516 ins_encode %{
17517 __ andpd($dst$$XMMRegister, ExternalAddress(double_signmask()));
17518 %}
17519 ins_pipe(pipe_slow);
17520 %}
17521
17522 instruct absD_reg_reg(vlRegD dst, vlRegD src) %{
17523 predicate(UseAVX > 0);
17524 match(Set dst (AbsD src));
17525 ins_cost(150);
17526 format %{ "vandpd $dst, $src, [0x7fffffffffffffff]\t"
17527 "# abs double by sign masking" %}
17528 ins_encode %{
17529 int vlen_enc = Assembler::AVX_128bit;
17530 __ vandpd($dst$$XMMRegister, $src$$XMMRegister,
17531 ExternalAddress(double_signmask()), vlen_enc);
17532 %}
17533 ins_pipe(pipe_slow);
17534 %}
17535
17536 instruct negF_reg(regF dst) %{
17537 predicate(UseAVX == 0);
17538 match(Set dst (NegF dst));
17539 ins_cost(150);
17540 format %{ "xorps $dst, [0x80000000]\t# neg float by sign flipping" %}
17541 ins_encode %{
17542 __ xorps($dst$$XMMRegister, ExternalAddress(float_signflip()));
17543 %}
17544 ins_pipe(pipe_slow);
17545 %}
17546
17547 instruct negF_reg_reg(vlRegF dst, vlRegF src) %{
17548 predicate(UseAVX > 0);
17549 match(Set dst (NegF src));
17550 ins_cost(150);
17551 format %{ "vnegatess $dst, $src, [0x80000000]\t# neg float by sign flipping" %}
17552 ins_encode %{
17553 __ vnegatess($dst$$XMMRegister, $src$$XMMRegister,
17554 ExternalAddress(float_signflip()));
17555 %}
17556 ins_pipe(pipe_slow);
17557 %}
17558
17559 instruct negD_reg(regD dst) %{
17560 predicate(UseAVX == 0);
17561 match(Set dst (NegD dst));
17562 ins_cost(150);
17563 format %{ "xorpd $dst, [0x8000000000000000]\t"
17564 "# neg double by sign flipping" %}
17565 ins_encode %{
17566 __ xorpd($dst$$XMMRegister, ExternalAddress(double_signflip()));
17567 %}
17568 ins_pipe(pipe_slow);
17569 %}
17570
17571 instruct negD_reg_reg(vlRegD dst, vlRegD src) %{
17572 predicate(UseAVX > 0);
17573 match(Set dst (NegD src));
17574 ins_cost(150);
17575 format %{ "vnegatesd $dst, $src, [0x8000000000000000]\t"
17576 "# neg double by sign flipping" %}
17577 ins_encode %{
17578 __ vnegatesd($dst$$XMMRegister, $src$$XMMRegister,
17579 ExternalAddress(double_signflip()));
17580 %}
17581 ins_pipe(pipe_slow);
17582 %}
17583
17584 // sqrtss instruction needs destination register to be pre initialized for best performance
17585 // Therefore only the instruct rule where the input is pre-loaded into dst register is defined below
17586 instruct sqrtF_reg(regF dst) %{
17587 match(Set dst (SqrtF dst));
17588 format %{ "sqrtss $dst, $dst" %}
17589 ins_encode %{
17590 __ sqrtss($dst$$XMMRegister, $dst$$XMMRegister);
17591 %}
17592 ins_pipe(pipe_slow);
17593 %}
17594
17595 // sqrtsd instruction needs destination register to be pre initialized for best performance
17596 // Therefore only the instruct rule where the input is pre-loaded into dst register is defined below
17597 instruct sqrtD_reg(regD dst) %{
17598 match(Set dst (SqrtD dst));
17599 format %{ "sqrtsd $dst, $dst" %}
17600 ins_encode %{
17601 __ sqrtsd($dst$$XMMRegister, $dst$$XMMRegister);
17602 %}
17603 ins_pipe(pipe_slow);
17604 %}
17605
17606 instruct convF2HF_reg_reg(rRegI dst, vlRegF src, vlRegF tmp) %{
17607 effect(TEMP tmp);
17608 match(Set dst (ConvF2HF src));
17609 ins_cost(125);
17610 format %{ "vcvtps2ph $dst,$src \t using $tmp as TEMP"%}
17611 ins_encode %{
17612 __ flt_to_flt16($dst$$Register, $src$$XMMRegister, $tmp$$XMMRegister);
17613 %}
17614 ins_pipe( pipe_slow );
17615 %}
17616
17617 instruct convF2HF_mem_reg(memory mem, regF src, kReg ktmp, rRegI rtmp) %{
17618 predicate((UseAVX > 2) && VM_Version::supports_avx512vl());
17619 effect(TEMP ktmp, TEMP rtmp);
17620 match(Set mem (StoreC mem (ConvF2HF src)));
17621 format %{ "evcvtps2ph $mem,$src \t using $ktmp and $rtmp as TEMP" %}
17622 ins_encode %{
17623 __ movl($rtmp$$Register, 0x1);
17624 __ kmovwl($ktmp$$KRegister, $rtmp$$Register);
17625 __ evcvtps2ph($mem$$Address, $ktmp$$KRegister, $src$$XMMRegister, 0x04, Assembler::AVX_128bit);
17626 %}
17627 ins_pipe( pipe_slow );
17628 %}
17629
17630 instruct vconvF2HF(vec dst, vec src) %{
17631 match(Set dst (VectorCastF2HF src));
17632 format %{ "vector_conv_F2HF $dst $src" %}
17633 ins_encode %{
17634 int vlen_enc = vector_length_encoding(this, $src);
17635 __ vcvtps2ph($dst$$XMMRegister, $src$$XMMRegister, 0x04, vlen_enc);
17636 %}
17637 ins_pipe( pipe_slow );
17638 %}
17639
17640 instruct vconvF2HF_mem_reg(memory mem, vec src) %{
17641 predicate(n->as_StoreVector()->memory_size() >= 16);
17642 match(Set mem (StoreVector mem (VectorCastF2HF src)));
17643 format %{ "vcvtps2ph $mem,$src" %}
17644 ins_encode %{
17645 int vlen_enc = vector_length_encoding(this, $src);
17646 __ vcvtps2ph($mem$$Address, $src$$XMMRegister, 0x04, vlen_enc);
17647 %}
17648 ins_pipe( pipe_slow );
17649 %}
17650
17651 instruct convHF2F_reg_reg(vlRegF dst, rRegI src) %{
17652 match(Set dst (ConvHF2F src));
17653 format %{ "vcvtph2ps $dst,$src" %}
17654 ins_encode %{
17655 __ flt16_to_flt($dst$$XMMRegister, $src$$Register);
17656 %}
17657 ins_pipe( pipe_slow );
17658 %}
17659
17660 instruct vconvHF2F_reg_mem(vec dst, memory mem) %{
17661 match(Set dst (VectorCastHF2F (LoadVector mem)));
17662 format %{ "vcvtph2ps $dst,$mem" %}
17663 ins_encode %{
17664 int vlen_enc = vector_length_encoding(this);
17665 __ vcvtph2ps($dst$$XMMRegister, $mem$$Address, vlen_enc);
17666 %}
17667 ins_pipe( pipe_slow );
17668 %}
17669
17670 instruct vconvHF2F(vec dst, vec src) %{
17671 match(Set dst (VectorCastHF2F src));
17672 ins_cost(125);
17673 format %{ "vector_conv_HF2F $dst,$src" %}
17674 ins_encode %{
17675 int vlen_enc = vector_length_encoding(this);
17676 __ vcvtph2ps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
17677 %}
17678 ins_pipe( pipe_slow );
17679 %}
17680
17681 // ---------------------------------------- VectorReinterpret ------------------------------------
17682 instruct reinterpret_mask(kReg dst) %{
17683 predicate(n->bottom_type()->isa_pvectmask() &&
17684 Matcher::vector_length(n) == Matcher::vector_length(n->in(1))); // dst == src
17685 match(Set dst (VectorReinterpret dst));
17686 ins_cost(125);
17687 format %{ "vector_reinterpret $dst\t!" %}
17688 ins_encode %{
17689 // empty
17690 %}
17691 ins_pipe( pipe_slow );
17692 %}
17693
17694 instruct reinterpret_mask_W2B(kReg dst, kReg src, vec xtmp) %{
17695 predicate(UseAVX > 2 && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
17696 n->bottom_type()->isa_pvectmask() &&
17697 n->in(1)->bottom_type()->isa_pvectmask() &&
17698 n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_SHORT &&
17699 n->bottom_type()->is_pvectmask()->element_basic_type() == T_BYTE); // dst == src
17700 match(Set dst (VectorReinterpret src));
17701 effect(TEMP xtmp);
17702 format %{ "vector_mask_reinterpret_W2B $dst $src\t!" %}
17703 ins_encode %{
17704 int src_sz = Matcher::vector_length(this, $src)*type2aelembytes(T_SHORT);
17705 int dst_sz = Matcher::vector_length(this)*type2aelembytes(T_BYTE);
17706 assert(src_sz == dst_sz , "src and dst size mismatch");
17707 int vlen_enc = vector_length_encoding(src_sz);
17708 __ evpmovm2w($xtmp$$XMMRegister, $src$$KRegister, vlen_enc);
17709 __ evpmovb2m($dst$$KRegister, $xtmp$$XMMRegister, vlen_enc);
17710 %}
17711 ins_pipe( pipe_slow );
17712 %}
17713
17714 instruct reinterpret_mask_D2B(kReg dst, kReg src, vec xtmp) %{
17715 predicate(UseAVX > 2 && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
17716 n->bottom_type()->isa_pvectmask() &&
17717 n->in(1)->bottom_type()->isa_pvectmask() &&
17718 (n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_INT ||
17719 n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_FLOAT) &&
17720 n->bottom_type()->is_pvectmask()->element_basic_type() == T_BYTE); // dst == src
17721 match(Set dst (VectorReinterpret src));
17722 effect(TEMP xtmp);
17723 format %{ "vector_mask_reinterpret_D2B $dst $src\t!" %}
17724 ins_encode %{
17725 int src_sz = Matcher::vector_length(this, $src)*type2aelembytes(T_INT);
17726 int dst_sz = Matcher::vector_length(this)*type2aelembytes(T_BYTE);
17727 assert(src_sz == dst_sz , "src and dst size mismatch");
17728 int vlen_enc = vector_length_encoding(src_sz);
17729 __ evpmovm2d($xtmp$$XMMRegister, $src$$KRegister, vlen_enc);
17730 __ evpmovb2m($dst$$KRegister, $xtmp$$XMMRegister, vlen_enc);
17731 %}
17732 ins_pipe( pipe_slow );
17733 %}
17734
17735 instruct reinterpret_mask_Q2B(kReg dst, kReg src, vec xtmp) %{
17736 predicate(UseAVX > 2 && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
17737 n->bottom_type()->isa_pvectmask() &&
17738 n->in(1)->bottom_type()->isa_pvectmask() &&
17739 (n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_LONG ||
17740 n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_DOUBLE) &&
17741 n->bottom_type()->is_pvectmask()->element_basic_type() == T_BYTE); // dst == src
17742 match(Set dst (VectorReinterpret src));
17743 effect(TEMP xtmp);
17744 format %{ "vector_mask_reinterpret_Q2B $dst $src\t!" %}
17745 ins_encode %{
17746 int src_sz = Matcher::vector_length(this, $src)*type2aelembytes(T_LONG);
17747 int dst_sz = Matcher::vector_length(this)*type2aelembytes(T_BYTE);
17748 assert(src_sz == dst_sz , "src and dst size mismatch");
17749 int vlen_enc = vector_length_encoding(src_sz);
17750 __ evpmovm2q($xtmp$$XMMRegister, $src$$KRegister, vlen_enc);
17751 __ evpmovb2m($dst$$KRegister, $xtmp$$XMMRegister, vlen_enc);
17752 %}
17753 ins_pipe( pipe_slow );
17754 %}
17755
17756 instruct reinterpret(vec dst) %{
17757 predicate(!n->bottom_type()->isa_pvectmask() &&
17758 Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1))); // dst == src
17759 match(Set dst (VectorReinterpret dst));
17760 ins_cost(125);
17761 format %{ "vector_reinterpret $dst\t!" %}
17762 ins_encode %{
17763 // empty
17764 %}
17765 ins_pipe( pipe_slow );
17766 %}
17767
17768 instruct reinterpret_expand(vec dst, vec src) %{
17769 predicate(UseAVX == 0 &&
17770 (Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
17771 match(Set dst (VectorReinterpret src));
17772 ins_cost(125);
17773 effect(TEMP dst);
17774 format %{ "vector_reinterpret_expand $dst,$src" %}
17775 ins_encode %{
17776 assert(Matcher::vector_length_in_bytes(this) <= 16, "required");
17777 assert(Matcher::vector_length_in_bytes(this, $src) <= 8, "required");
17778
17779 int src_vlen_in_bytes = Matcher::vector_length_in_bytes(this, $src);
17780 if (src_vlen_in_bytes == 4) {
17781 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_32_bit_mask()), noreg);
17782 } else {
17783 assert(src_vlen_in_bytes == 8, "");
17784 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_64_bit_mask()), noreg);
17785 }
17786 __ pand($dst$$XMMRegister, $src$$XMMRegister);
17787 %}
17788 ins_pipe( pipe_slow );
17789 %}
17790
17791 instruct vreinterpret_expand4(legVec dst, vec src) %{
17792 predicate(UseAVX > 0 &&
17793 !n->bottom_type()->isa_pvectmask() &&
17794 (Matcher::vector_length_in_bytes(n->in(1)) == 4) && // src
17795 (Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
17796 match(Set dst (VectorReinterpret src));
17797 ins_cost(125);
17798 format %{ "vector_reinterpret_expand $dst,$src" %}
17799 ins_encode %{
17800 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_32_bit_mask()), 0, noreg);
17801 %}
17802 ins_pipe( pipe_slow );
17803 %}
17804
17805
17806 instruct vreinterpret_expand(legVec dst, vec src) %{
17807 predicate(UseAVX > 0 &&
17808 !n->bottom_type()->isa_pvectmask() &&
17809 (Matcher::vector_length_in_bytes(n->in(1)) > 4) && // src
17810 (Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
17811 match(Set dst (VectorReinterpret src));
17812 ins_cost(125);
17813 format %{ "vector_reinterpret_expand $dst,$src\t!" %}
17814 ins_encode %{
17815 switch (Matcher::vector_length_in_bytes(this, $src)) {
17816 case 8: __ movq ($dst$$XMMRegister, $src$$XMMRegister); break;
17817 case 16: __ movdqu ($dst$$XMMRegister, $src$$XMMRegister); break;
17818 case 32: __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister); break;
17819 default: ShouldNotReachHere();
17820 }
17821 %}
17822 ins_pipe( pipe_slow );
17823 %}
17824
17825 instruct reinterpret_shrink(vec dst, legVec src) %{
17826 predicate(!n->bottom_type()->isa_pvectmask() &&
17827 Matcher::vector_length_in_bytes(n->in(1)) > Matcher::vector_length_in_bytes(n)); // src > dst
17828 match(Set dst (VectorReinterpret src));
17829 ins_cost(125);
17830 format %{ "vector_reinterpret_shrink $dst,$src\t!" %}
17831 ins_encode %{
17832 switch (Matcher::vector_length_in_bytes(this)) {
17833 case 4: __ movfltz($dst$$XMMRegister, $src$$XMMRegister); break;
17834 case 8: __ movq ($dst$$XMMRegister, $src$$XMMRegister); break;
17835 case 16: __ movdqu ($dst$$XMMRegister, $src$$XMMRegister); break;
17836 case 32: __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister); break;
17837 default: ShouldNotReachHere();
17838 }
17839 %}
17840 ins_pipe( pipe_slow );
17841 %}
17842
17843 // ----------------------------------------------------------------------------------------------------
17844
17845 instruct roundD_reg(legRegD dst, legRegD src, immU8 rmode) %{
17846 match(Set dst (RoundDoubleMode src rmode));
17847 format %{ "roundsd $dst,$src" %}
17848 ins_cost(150);
17849 ins_encode %{
17850 assert(UseSSE >= 4, "required");
17851 if ((UseAVX == 0) && ($dst$$XMMRegister != $src$$XMMRegister)) {
17852 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
17853 }
17854 __ roundsd($dst$$XMMRegister, $src$$XMMRegister, $rmode$$constant);
17855 %}
17856 ins_pipe(pipe_slow);
17857 %}
17858
17859 instruct roundD_imm(legRegD dst, immD con, immU8 rmode) %{
17860 match(Set dst (RoundDoubleMode con rmode));
17861 format %{ "roundsd $dst,[$constantaddress]\t# load from constant table: double=$con" %}
17862 ins_cost(150);
17863 ins_encode %{
17864 assert(UseSSE >= 4, "required");
17865 __ roundsd($dst$$XMMRegister, $constantaddress($con), $rmode$$constant, noreg);
17866 %}
17867 ins_pipe(pipe_slow);
17868 %}
17869
17870 instruct vroundD_reg(legVec dst, legVec src, immU8 rmode) %{
17871 predicate(Matcher::vector_length(n) < 8);
17872 match(Set dst (RoundDoubleModeV src rmode));
17873 format %{ "vroundpd $dst,$src,$rmode\t! round packedD" %}
17874 ins_encode %{
17875 assert(UseAVX > 0, "required");
17876 int vlen_enc = vector_length_encoding(this);
17877 __ vroundpd($dst$$XMMRegister, $src$$XMMRegister, $rmode$$constant, vlen_enc);
17878 %}
17879 ins_pipe( pipe_slow );
17880 %}
17881
17882 instruct vround8D_reg(vec dst, vec src, immU8 rmode) %{
17883 predicate(Matcher::vector_length(n) == 8);
17884 match(Set dst (RoundDoubleModeV src rmode));
17885 format %{ "vrndscalepd $dst,$src,$rmode\t! round packed8D" %}
17886 ins_encode %{
17887 assert(UseAVX > 2, "required");
17888 __ vrndscalepd($dst$$XMMRegister, $src$$XMMRegister, $rmode$$constant, Assembler::AVX_512bit);
17889 %}
17890 ins_pipe( pipe_slow );
17891 %}
17892
17893 instruct vroundD_mem(legVec dst, memory mem, immU8 rmode) %{
17894 predicate(Matcher::vector_length(n) < 8);
17895 match(Set dst (RoundDoubleModeV (LoadVector mem) rmode));
17896 format %{ "vroundpd $dst, $mem, $rmode\t! round packedD" %}
17897 ins_encode %{
17898 assert(UseAVX > 0, "required");
17899 int vlen_enc = vector_length_encoding(this);
17900 __ vroundpd($dst$$XMMRegister, $mem$$Address, $rmode$$constant, vlen_enc);
17901 %}
17902 ins_pipe( pipe_slow );
17903 %}
17904
17905 instruct vround8D_mem(vec dst, memory mem, immU8 rmode) %{
17906 predicate(Matcher::vector_length(n) == 8);
17907 match(Set dst (RoundDoubleModeV (LoadVector mem) rmode));
17908 format %{ "vrndscalepd $dst,$mem,$rmode\t! round packed8D" %}
17909 ins_encode %{
17910 assert(UseAVX > 2, "required");
17911 __ vrndscalepd($dst$$XMMRegister, $mem$$Address, $rmode$$constant, Assembler::AVX_512bit);
17912 %}
17913 ins_pipe( pipe_slow );
17914 %}
17915
17916 instruct onspinwait() %{
17917 match(OnSpinWait);
17918 ins_cost(200);
17919
17920 format %{
17921 $$template
17922 $$emit$$"pause\t! membar_onspinwait"
17923 %}
17924 ins_encode %{
17925 __ pause();
17926 %}
17927 ins_pipe(pipe_slow);
17928 %}
17929
17930 // a * b + c
17931 instruct fmaD_reg(regD a, regD b, regD c) %{
17932 match(Set c (FmaD c (Binary a b)));
17933 format %{ "fmasd $a,$b,$c\t# $c = $a * $b + $c" %}
17934 ins_cost(150);
17935 ins_encode %{
17936 assert(UseFMA, "Needs FMA instructions support.");
17937 __ fmad($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister);
17938 %}
17939 ins_pipe( pipe_slow );
17940 %}
17941
17942 // a * b + c
17943 instruct fmaF_reg(regF a, regF b, regF c) %{
17944 match(Set c (FmaF c (Binary a b)));
17945 format %{ "fmass $a,$b,$c\t# $c = $a * $b + $c" %}
17946 ins_cost(150);
17947 ins_encode %{
17948 assert(UseFMA, "Needs FMA instructions support.");
17949 __ fmaf($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister);
17950 %}
17951 ins_pipe( pipe_slow );
17952 %}
17953
17954 // ====================VECTOR INSTRUCTIONS=====================================
17955
17956 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
17957 instruct MoveVec2Leg(legVec dst, vec src) %{
17958 match(Set dst src);
17959 format %{ "" %}
17960 ins_encode %{
17961 ShouldNotReachHere();
17962 %}
17963 ins_pipe( fpu_reg_reg );
17964 %}
17965
17966 instruct MoveLeg2Vec(vec dst, legVec src) %{
17967 match(Set dst src);
17968 format %{ "" %}
17969 ins_encode %{
17970 ShouldNotReachHere();
17971 %}
17972 ins_pipe( fpu_reg_reg );
17973 %}
17974
17975 // ============================================================================
17976
17977 // Load vectors generic operand pattern
17978 instruct loadV(vec dst, memory mem) %{
17979 match(Set dst (LoadVector mem));
17980 ins_cost(125);
17981 format %{ "load_vector $dst,$mem" %}
17982 ins_encode %{
17983 BasicType bt = Matcher::vector_element_basic_type(this);
17984 __ load_vector(bt, $dst$$XMMRegister, $mem$$Address, Matcher::vector_length_in_bytes(this));
17985 %}
17986 ins_pipe( pipe_slow );
17987 %}
17988
17989 // Store vectors generic operand pattern.
17990 instruct storeV(memory mem, vec src) %{
17991 match(Set mem (StoreVector mem src));
17992 ins_cost(145);
17993 format %{ "store_vector $mem,$src\n\t" %}
17994 ins_encode %{
17995 switch (Matcher::vector_length_in_bytes(this, $src)) {
17996 case 4: __ movdl ($mem$$Address, $src$$XMMRegister); break;
17997 case 8: __ movq ($mem$$Address, $src$$XMMRegister); break;
17998 case 16: __ movdqu ($mem$$Address, $src$$XMMRegister); break;
17999 case 32: __ vmovdqu ($mem$$Address, $src$$XMMRegister); break;
18000 case 64: __ evmovdqul($mem$$Address, $src$$XMMRegister, Assembler::AVX_512bit); break;
18001 default: ShouldNotReachHere();
18002 }
18003 %}
18004 ins_pipe( pipe_slow );
18005 %}
18006
18007 // ---------------------------------------- Gather ------------------------------------
18008
18009 // Gather BYTE, SHORT, INT, LONG, FLOAT, DOUBLE
18010
18011 instruct gather(legVec dst, memory mem, legVec idx, rRegP tmp, legVec mask) %{
18012 predicate(!VM_Version::supports_avx512vl() && !is_subword_type(Matcher::vector_element_basic_type(n)) &&
18013 Matcher::vector_length_in_bytes(n) <= 32);
18014 match(Set dst (LoadVectorGather mem idx));
18015 effect(TEMP dst, TEMP tmp, TEMP mask);
18016 format %{ "load_vector_gather $dst, $mem, $idx\t! using $tmp and $mask as TEMP" %}
18017 ins_encode %{
18018 int vlen_enc = vector_length_encoding(this);
18019 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18020 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18021 __ vpcmpeqd($mask$$XMMRegister, $mask$$XMMRegister, $mask$$XMMRegister, vlen_enc);
18022 __ lea($tmp$$Register, $mem$$Address);
18023 __ vgather(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx$$XMMRegister, $mask$$XMMRegister, vlen_enc);
18024 %}
18025 ins_pipe( pipe_slow );
18026 %}
18027
18028
18029 instruct evgather(vec dst, memory mem, vec idx, rRegP tmp, kReg ktmp) %{
18030 predicate((VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64) &&
18031 !is_subword_type(Matcher::vector_element_basic_type(n)));
18032 match(Set dst (LoadVectorGather mem idx));
18033 effect(TEMP dst, TEMP tmp, TEMP ktmp);
18034 format %{ "load_vector_gather $dst, $mem, $idx\t! using $tmp and ktmp as TEMP" %}
18035 ins_encode %{
18036 int vlen_enc = vector_length_encoding(this);
18037 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18038 __ kxnorwl($ktmp$$KRegister, $ktmp$$KRegister, $ktmp$$KRegister);
18039 __ lea($tmp$$Register, $mem$$Address);
18040 __ evgather(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $tmp$$Register, $idx$$XMMRegister, vlen_enc);
18041 %}
18042 ins_pipe( pipe_slow );
18043 %}
18044
18045 instruct evgather_masked(vec dst, memory mem, vec idx, kReg mask, kReg ktmp, rRegP tmp) %{
18046 predicate((VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64) &&
18047 !is_subword_type(Matcher::vector_element_basic_type(n)));
18048 match(Set dst (LoadVectorGatherMasked mem (Binary idx mask)));
18049 effect(TEMP_DEF dst, TEMP tmp, TEMP ktmp);
18050 format %{ "load_vector_gather_masked $dst, $mem, $idx, $mask\t! using $tmp and ktmp as TEMP" %}
18051 ins_encode %{
18052 assert(UseAVX > 2, "sanity");
18053 int vlen_enc = vector_length_encoding(this);
18054 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18055 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18056 // Note: Since gather instruction partially updates the opmask register used
18057 // for predication hense moving mask operand to a temporary.
18058 __ kmovwl($ktmp$$KRegister, $mask$$KRegister);
18059 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18060 __ lea($tmp$$Register, $mem$$Address);
18061 __ evgather(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $tmp$$Register, $idx$$XMMRegister, vlen_enc);
18062 %}
18063 ins_pipe( pipe_slow );
18064 %}
18065
18066 instruct vgather_subwordLE8B(vec dst, memory mem, rRegP idx_base, rRegP tmp, rRegI rtmp) %{
18067 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) <= 8);
18068 match(Set dst (LoadVectorGather mem idx_base));
18069 effect(TEMP tmp, TEMP rtmp);
18070 format %{ "vector_gatherLE8 $dst, $mem, $idx_base\t! using $tmp and $rtmp as TEMP" %}
18071 ins_encode %{
18072 int vlen_enc = vector_length_encoding(this);
18073 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18074 __ lea($tmp$$Register, $mem$$Address);
18075 __ vgather8b(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base$$Register, $rtmp$$Register, vlen_enc);
18076 %}
18077 ins_pipe( pipe_slow );
18078 %}
18079
18080 instruct vgather_subwordGT8B(vec dst, memory mem, rRegP idx_base, rRegP tmp, rRegP idx_base_temp,
18081 vec xtmp1, vec xtmp2, vec xtmp3, rRegI rtmp, rRegI length, rFlagsReg cr) %{
18082 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) > 8);
18083 match(Set dst (LoadVectorGather mem idx_base));
18084 effect(TEMP_DEF dst, TEMP tmp, TEMP idx_base_temp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp, TEMP length, KILL cr);
18085 format %{ "vector_gatherGT8 $dst, $mem, $idx_base\t! using $tmp, $idx_base_temp, $xtmp1, $xtmp2, $xtmp3, $rtmp and $length as TEMP" %}
18086 ins_encode %{
18087 int vlen_enc = vector_length_encoding(this);
18088 int vector_len = Matcher::vector_length(this);
18089 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18090 __ lea($tmp$$Register, $mem$$Address);
18091 __ movptr($idx_base_temp$$Register, $idx_base$$Register);
18092 __ vgather_subword(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base_temp$$Register, noreg, $xtmp1$$XMMRegister,
18093 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, noreg, $length$$Register, vector_len, vlen_enc);
18094 %}
18095 ins_pipe( pipe_slow );
18096 %}
18097
18098 instruct vgather_masked_subwordLE8B_avx3(vec dst, memory mem, rRegP idx_base, kReg mask, rRegL mask_idx, rRegP tmp, rRegI rtmp, rRegL rtmp2, rFlagsReg cr) %{
18099 predicate(VM_Version::supports_avx512bw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) <= 8);
18100 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18101 effect(TEMP mask_idx, TEMP tmp, TEMP rtmp, TEMP rtmp2, KILL cr);
18102 format %{ "vector_masked_gatherLE8 $dst, $mem, $idx_base, $mask\t! using $mask_idx, $tmp, $rtmp and $rtmp2 as TEMP" %}
18103 ins_encode %{
18104 int vlen_enc = vector_length_encoding(this);
18105 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18106 __ xorq($mask_idx$$Register, $mask_idx$$Register);
18107 __ lea($tmp$$Register, $mem$$Address);
18108 __ kmovql($rtmp2$$Register, $mask$$KRegister);
18109 __ vgather8b_masked(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base$$Register, $rtmp2$$Register, $mask_idx$$Register, $rtmp$$Register, vlen_enc);
18110 %}
18111 ins_pipe( pipe_slow );
18112 %}
18113
18114 instruct vgather_masked_subwordGT8B_avx3(vec dst, memory mem, rRegP idx_base, kReg mask, rRegP tmp, rRegP idx_base_temp,
18115 vec xtmp1, vec xtmp2, vec xtmp3, rRegI rtmp, rRegL rtmp2, rRegL mask_idx, rRegI length, rFlagsReg cr) %{
18116 predicate(VM_Version::supports_avx512bw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) > 8);
18117 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18118 effect(TEMP_DEF dst, TEMP tmp, TEMP idx_base_temp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp, TEMP rtmp2, TEMP mask_idx, TEMP length, KILL cr);
18119 format %{ "vector_gatherGT8_masked $dst, $mem, $idx_base, $mask\t! using $tmp, $idx_base_temp, $xtmp1, $xtmp2, $xtmp3, $rtmp, $rtmp2, $mask_idx and $length as TEMP" %}
18120 ins_encode %{
18121 int vlen_enc = vector_length_encoding(this);
18122 int vector_len = Matcher::vector_length(this);
18123 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18124 __ xorq($mask_idx$$Register, $mask_idx$$Register);
18125 __ lea($tmp$$Register, $mem$$Address);
18126 __ movptr($idx_base_temp$$Register, $idx_base$$Register);
18127 __ kmovql($rtmp2$$Register, $mask$$KRegister);
18128 __ vgather_subword(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base_temp$$Register, $rtmp2$$Register, $xtmp1$$XMMRegister,
18129 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, $mask_idx$$Register, $length$$Register, vector_len, vlen_enc);
18130 %}
18131 ins_pipe( pipe_slow );
18132 %}
18133
18134 instruct vgather_masked_subwordLE8B_avx2(vec dst, memory mem, rRegP idx_base, vec mask, rRegI mask_idx, rRegP tmp, rRegI rtmp, rRegI rtmp2, rFlagsReg cr) %{
18135 predicate(!VM_Version::supports_avx512vlbw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) <= 8);
18136 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18137 effect(TEMP mask_idx, TEMP tmp, TEMP rtmp, TEMP rtmp2, KILL cr);
18138 format %{ "vector_masked_gatherLE8 $dst, $mem, $idx_base, $mask\t! using $mask_idx, $tmp, $rtmp and $rtmp2 as TEMP" %}
18139 ins_encode %{
18140 int vlen_enc = vector_length_encoding(this);
18141 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18142 __ lea($tmp$$Register, $mem$$Address);
18143 __ vpmovmskb($rtmp2$$Register, $mask$$XMMRegister, vlen_enc);
18144 if (elem_bt == T_SHORT) {
18145 __ movl($mask_idx$$Register, 0x55555555);
18146 __ pextl($rtmp2$$Register, $rtmp2$$Register, $mask_idx$$Register);
18147 }
18148 __ xorl($mask_idx$$Register, $mask_idx$$Register);
18149 __ vgather8b_masked(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base$$Register, $rtmp2$$Register, $mask_idx$$Register, $rtmp$$Register, vlen_enc);
18150 %}
18151 ins_pipe( pipe_slow );
18152 %}
18153
18154 instruct vgather_masked_subwordGT8B_avx2(vec dst, memory mem, rRegP idx_base, vec mask, rRegP tmp, rRegP idx_base_temp,
18155 vec xtmp1, vec xtmp2, vec xtmp3, rRegI rtmp, rRegI rtmp2, rRegI mask_idx, rRegI length, rFlagsReg cr) %{
18156 predicate(!VM_Version::supports_avx512vlbw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) > 8);
18157 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18158 effect(TEMP_DEF dst, TEMP tmp, TEMP idx_base_temp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp, TEMP rtmp2, TEMP mask_idx, TEMP length, KILL cr);
18159 format %{ "vector_gatherGT8_masked $dst, $mem, $idx_base, $mask\t! using $tmp, $idx_base_temp, $xtmp1, $xtmp2, $xtmp3, $rtmp, $rtmp2, $mask_idx and $length as TEMP" %}
18160 ins_encode %{
18161 int vlen_enc = vector_length_encoding(this);
18162 int vector_len = Matcher::vector_length(this);
18163 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18164 __ lea($tmp$$Register, $mem$$Address);
18165 __ movptr($idx_base_temp$$Register, $idx_base$$Register);
18166 __ vpmovmskb($rtmp2$$Register, $mask$$XMMRegister, vlen_enc);
18167 if (elem_bt == T_SHORT) {
18168 __ movl($mask_idx$$Register, 0x55555555);
18169 __ pextl($rtmp2$$Register, $rtmp2$$Register, $mask_idx$$Register);
18170 }
18171 __ xorl($mask_idx$$Register, $mask_idx$$Register);
18172 __ vgather_subword(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base_temp$$Register, $rtmp2$$Register, $xtmp1$$XMMRegister,
18173 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, $mask_idx$$Register, $length$$Register, vector_len, vlen_enc);
18174 %}
18175 ins_pipe( pipe_slow );
18176 %}
18177
18178 // ====================Scatter=======================================
18179
18180 // Scatter INT, LONG, FLOAT, DOUBLE
18181
18182 instruct scatter(memory mem, vec src, vec idx, rRegP tmp, kReg ktmp) %{
18183 predicate(UseAVX > 2);
18184 match(Set mem (StoreVectorScatter mem (Binary src idx)));
18185 effect(TEMP tmp, TEMP ktmp);
18186 format %{ "store_vector_scatter $mem, $idx, $src\t! using k2 and $tmp as TEMP" %}
18187 ins_encode %{
18188 int vlen_enc = vector_length_encoding(this, $src);
18189 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
18190
18191 assert(Matcher::vector_length_in_bytes(this, $src) >= 16, "sanity");
18192 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18193
18194 __ kmovwl($ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), noreg);
18195 __ lea($tmp$$Register, $mem$$Address);
18196 __ evscatter(elem_bt, $tmp$$Register, $idx$$XMMRegister, $ktmp$$KRegister, $src$$XMMRegister, vlen_enc);
18197 %}
18198 ins_pipe( pipe_slow );
18199 %}
18200
18201 instruct scatter_masked(memory mem, vec src, vec idx, kReg mask, kReg ktmp, rRegP tmp) %{
18202 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx mask))));
18203 effect(TEMP tmp, TEMP ktmp);
18204 format %{ "store_vector_scatter_masked $mem, $idx, $src, $mask\t!" %}
18205 ins_encode %{
18206 int vlen_enc = vector_length_encoding(this, $src);
18207 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
18208 assert(Matcher::vector_length_in_bytes(this, $src) >= 16, "sanity");
18209 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18210 // Note: Since scatter instruction partially updates the opmask register used
18211 // for predication hense moving mask operand to a temporary.
18212 __ kmovwl($ktmp$$KRegister, $mask$$KRegister);
18213 __ lea($tmp$$Register, $mem$$Address);
18214 __ evscatter(elem_bt, $tmp$$Register, $idx$$XMMRegister, $ktmp$$KRegister, $src$$XMMRegister, vlen_enc);
18215 %}
18216 ins_pipe( pipe_slow );
18217 %}
18218
18219 // ====================REPLICATE=======================================
18220
18221 // Replicate byte scalar to be vector
18222 instruct vReplB_reg(vec dst, rRegI src) %{
18223 predicate(Matcher::vector_element_basic_type(n) == T_BYTE);
18224 match(Set dst (Replicate src));
18225 format %{ "replicateB $dst,$src" %}
18226 ins_encode %{
18227 uint vlen = Matcher::vector_length(this);
18228 if (UseAVX >= 2) {
18229 int vlen_enc = vector_length_encoding(this);
18230 if (vlen == 64 || VM_Version::supports_avx512vlbw()) { // AVX512VL for <512bit operands
18231 assert(VM_Version::supports_avx512bw(), "required"); // 512-bit byte vectors assume AVX512BW
18232 __ evpbroadcastb($dst$$XMMRegister, $src$$Register, vlen_enc);
18233 } else {
18234 __ movdl($dst$$XMMRegister, $src$$Register);
18235 __ vpbroadcastb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18236 }
18237 } else {
18238 assert(UseAVX < 2, "");
18239 __ movdl($dst$$XMMRegister, $src$$Register);
18240 __ punpcklbw($dst$$XMMRegister, $dst$$XMMRegister);
18241 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18242 if (vlen >= 16) {
18243 assert(vlen == 16, "");
18244 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18245 }
18246 }
18247 %}
18248 ins_pipe( pipe_slow );
18249 %}
18250
18251 instruct ReplB_mem(vec dst, memory mem) %{
18252 predicate(UseAVX >= 2 && Matcher::vector_element_basic_type(n) == T_BYTE);
18253 match(Set dst (Replicate (LoadB mem)));
18254 format %{ "replicateB $dst,$mem" %}
18255 ins_encode %{
18256 int vlen_enc = vector_length_encoding(this);
18257 __ vpbroadcastb($dst$$XMMRegister, $mem$$Address, vlen_enc);
18258 %}
18259 ins_pipe( pipe_slow );
18260 %}
18261
18262 // ====================ReplicateS=======================================
18263
18264 instruct vReplS_reg(vec dst, rRegI src) %{
18265 predicate(Matcher::vector_element_basic_type(n) == T_SHORT);
18266 match(Set dst (Replicate src));
18267 format %{ "replicateS $dst,$src" %}
18268 ins_encode %{
18269 uint vlen = Matcher::vector_length(this);
18270 int vlen_enc = vector_length_encoding(this);
18271 if (UseAVX >= 2) {
18272 if (vlen == 32 || VM_Version::supports_avx512vlbw()) { // AVX512VL for <512bit operands
18273 assert(VM_Version::supports_avx512bw(), "required"); // 512-bit short vectors assume AVX512BW
18274 __ evpbroadcastw($dst$$XMMRegister, $src$$Register, vlen_enc);
18275 } else {
18276 __ movdl($dst$$XMMRegister, $src$$Register);
18277 __ vpbroadcastw($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18278 }
18279 } else {
18280 assert(UseAVX < 2, "");
18281 __ movdl($dst$$XMMRegister, $src$$Register);
18282 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18283 if (vlen >= 8) {
18284 assert(vlen == 8, "");
18285 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18286 }
18287 }
18288 %}
18289 ins_pipe( pipe_slow );
18290 %}
18291
18292 instruct ReplHF_imm(vec dst, immH con, rRegI rtmp) %{
18293 match(Set dst (Replicate con));
18294 effect(TEMP rtmp);
18295 format %{ "replicateHF $dst, $con \t! using $rtmp as TEMP" %}
18296 ins_encode %{
18297 int vlen_enc = vector_length_encoding(this);
18298 BasicType bt = Matcher::vector_element_basic_type(this);
18299 assert(VM_Version::supports_avx512_fp16() && bt == T_SHORT, "");
18300 __ movl($rtmp$$Register, $con$$constant);
18301 __ evpbroadcastw($dst$$XMMRegister, $rtmp$$Register, vlen_enc);
18302 %}
18303 ins_pipe( pipe_slow );
18304 %}
18305
18306 instruct ReplHF_reg(vec dst, regF src, rRegI rtmp) %{
18307 predicate(VM_Version::supports_avx512_fp16() && Matcher::vector_element_basic_type(n) == T_SHORT);
18308 match(Set dst (Replicate src));
18309 effect(TEMP rtmp);
18310 format %{ "replicateHF $dst, $src \t! using $rtmp as TEMP" %}
18311 ins_encode %{
18312 int vlen_enc = vector_length_encoding(this);
18313 __ evmovw($rtmp$$Register, $src$$XMMRegister);
18314 __ evpbroadcastw($dst$$XMMRegister, $rtmp$$Register, vlen_enc);
18315 %}
18316 ins_pipe( pipe_slow );
18317 %}
18318
18319 instruct ReplS_mem(vec dst, memory mem) %{
18320 predicate(UseAVX >= 2 && Matcher::vector_element_basic_type(n) == T_SHORT);
18321 match(Set dst (Replicate (LoadS mem)));
18322 format %{ "replicateS $dst,$mem" %}
18323 ins_encode %{
18324 int vlen_enc = vector_length_encoding(this);
18325 __ vpbroadcastw($dst$$XMMRegister, $mem$$Address, vlen_enc);
18326 %}
18327 ins_pipe( pipe_slow );
18328 %}
18329
18330 // ====================ReplicateI=======================================
18331
18332 instruct ReplI_reg(vec dst, rRegI src) %{
18333 predicate(Matcher::vector_element_basic_type(n) == T_INT);
18334 match(Set dst (Replicate src));
18335 format %{ "replicateI $dst,$src" %}
18336 ins_encode %{
18337 uint vlen = Matcher::vector_length(this);
18338 int vlen_enc = vector_length_encoding(this);
18339 if (vlen == 16 || VM_Version::supports_avx512vl()) { // AVX512VL for <512bit operands
18340 __ evpbroadcastd($dst$$XMMRegister, $src$$Register, vlen_enc);
18341 } else if (VM_Version::supports_avx2()) {
18342 __ movdl($dst$$XMMRegister, $src$$Register);
18343 __ vpbroadcastd($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18344 } else {
18345 __ movdl($dst$$XMMRegister, $src$$Register);
18346 __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18347 }
18348 %}
18349 ins_pipe( pipe_slow );
18350 %}
18351
18352 instruct ReplI_mem(vec dst, memory mem) %{
18353 predicate(Matcher::vector_element_basic_type(n) == T_INT);
18354 match(Set dst (Replicate (LoadI mem)));
18355 format %{ "replicateI $dst,$mem" %}
18356 ins_encode %{
18357 int vlen_enc = vector_length_encoding(this);
18358 if (VM_Version::supports_avx2()) {
18359 __ vpbroadcastd($dst$$XMMRegister, $mem$$Address, vlen_enc);
18360 } else if (VM_Version::supports_avx()) {
18361 __ vbroadcastss($dst$$XMMRegister, $mem$$Address, vlen_enc);
18362 } else {
18363 __ movdl($dst$$XMMRegister, $mem$$Address);
18364 __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18365 }
18366 %}
18367 ins_pipe( pipe_slow );
18368 %}
18369
18370 instruct ReplI_imm(vec dst, immI con) %{
18371 predicate(Matcher::is_non_long_integral_vector(n));
18372 match(Set dst (Replicate con));
18373 format %{ "replicateI $dst,$con" %}
18374 ins_encode %{
18375 InternalAddress addr = $constantaddress(vreplicate_imm(Matcher::vector_element_basic_type(this), $con$$constant,
18376 (VM_Version::supports_sse3() ? (VM_Version::supports_avx() ? 4 : 8) : 16) /
18377 type2aelembytes(Matcher::vector_element_basic_type(this))));
18378 BasicType bt = Matcher::vector_element_basic_type(this);
18379 int vlen = Matcher::vector_length_in_bytes(this);
18380 __ load_constant_vector(bt, $dst$$XMMRegister, addr, vlen);
18381 %}
18382 ins_pipe( pipe_slow );
18383 %}
18384
18385 // Replicate scalar zero to be vector
18386 instruct ReplI_zero(vec dst, immI_0 zero) %{
18387 predicate(Matcher::is_non_long_integral_vector(n));
18388 match(Set dst (Replicate zero));
18389 format %{ "replicateI $dst,$zero" %}
18390 ins_encode %{
18391 int vlen_enc = vector_length_encoding(this);
18392 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vl()) {
18393 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18394 } else {
18395 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
18396 }
18397 %}
18398 ins_pipe( fpu_reg_reg );
18399 %}
18400
18401 instruct ReplI_M1(vec dst, immI_M1 con) %{
18402 predicate(Matcher::is_non_long_integral_vector(n));
18403 match(Set dst (Replicate con));
18404 format %{ "vallones $dst" %}
18405 ins_encode %{
18406 int vector_len = vector_length_encoding(this);
18407 __ vallones($dst$$XMMRegister, vector_len);
18408 %}
18409 ins_pipe( pipe_slow );
18410 %}
18411
18412 // ====================ReplicateL=======================================
18413
18414 // Replicate long (8 byte) scalar to be vector
18415 instruct ReplL_reg(vec dst, rRegL src) %{
18416 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18417 match(Set dst (Replicate src));
18418 format %{ "replicateL $dst,$src" %}
18419 ins_encode %{
18420 int vlen = Matcher::vector_length(this);
18421 int vlen_enc = vector_length_encoding(this);
18422 if (vlen == 8 || VM_Version::supports_avx512vl()) { // AVX512VL for <512bit operands
18423 __ evpbroadcastq($dst$$XMMRegister, $src$$Register, vlen_enc);
18424 } else if (VM_Version::supports_avx2()) {
18425 __ movdq($dst$$XMMRegister, $src$$Register);
18426 __ vpbroadcastq($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18427 } else {
18428 __ movdq($dst$$XMMRegister, $src$$Register);
18429 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18430 }
18431 %}
18432 ins_pipe( pipe_slow );
18433 %}
18434
18435 instruct ReplL_mem(vec dst, memory mem) %{
18436 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18437 match(Set dst (Replicate (LoadL mem)));
18438 format %{ "replicateL $dst,$mem" %}
18439 ins_encode %{
18440 int vlen_enc = vector_length_encoding(this);
18441 if (VM_Version::supports_avx2()) {
18442 __ vpbroadcastq($dst$$XMMRegister, $mem$$Address, vlen_enc);
18443 } else if (VM_Version::supports_sse3()) {
18444 __ movddup($dst$$XMMRegister, $mem$$Address);
18445 } else {
18446 __ movq($dst$$XMMRegister, $mem$$Address);
18447 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18448 }
18449 %}
18450 ins_pipe( pipe_slow );
18451 %}
18452
18453 // Replicate long (8 byte) scalar immediate to be vector by loading from const table.
18454 instruct ReplL_imm(vec dst, immL con) %{
18455 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18456 match(Set dst (Replicate con));
18457 format %{ "replicateL $dst,$con" %}
18458 ins_encode %{
18459 InternalAddress addr = $constantaddress(vreplicate_imm(T_LONG, $con$$constant, VM_Version::supports_sse3() ? 1 : 2));
18460 int vlen = Matcher::vector_length_in_bytes(this);
18461 __ load_constant_vector(T_LONG, $dst$$XMMRegister, addr, vlen);
18462 %}
18463 ins_pipe( pipe_slow );
18464 %}
18465
18466 instruct ReplL_zero(vec dst, immL0 zero) %{
18467 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18468 match(Set dst (Replicate zero));
18469 format %{ "replicateL $dst,$zero" %}
18470 ins_encode %{
18471 int vlen_enc = vector_length_encoding(this);
18472 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vl()) {
18473 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18474 } else {
18475 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
18476 }
18477 %}
18478 ins_pipe( fpu_reg_reg );
18479 %}
18480
18481 instruct ReplL_M1(vec dst, immL_M1 con) %{
18482 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18483 match(Set dst (Replicate con));
18484 format %{ "vallones $dst" %}
18485 ins_encode %{
18486 int vector_len = vector_length_encoding(this);
18487 __ vallones($dst$$XMMRegister, vector_len);
18488 %}
18489 ins_pipe( pipe_slow );
18490 %}
18491
18492 // ====================ReplicateF=======================================
18493
18494 instruct vReplF_reg(vec dst, vlRegF src) %{
18495 predicate(UseAVX > 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
18496 match(Set dst (Replicate src));
18497 format %{ "replicateF $dst,$src" %}
18498 ins_encode %{
18499 uint vlen = Matcher::vector_length(this);
18500 int vlen_enc = vector_length_encoding(this);
18501 if (vlen <= 4) {
18502 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 0x00, Assembler::AVX_128bit);
18503 } else if (VM_Version::supports_avx2()) {
18504 __ vbroadcastss($dst$$XMMRegister, $src$$XMMRegister, vlen_enc); // reg-to-reg variant requires AVX2
18505 } else {
18506 assert(vlen == 8, "sanity");
18507 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 0x00, Assembler::AVX_128bit);
18508 __ vinsertf128_high($dst$$XMMRegister, $dst$$XMMRegister);
18509 }
18510 %}
18511 ins_pipe( pipe_slow );
18512 %}
18513
18514 instruct ReplF_reg(vec dst, vlRegF src) %{
18515 predicate(UseAVX == 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
18516 match(Set dst (Replicate src));
18517 format %{ "replicateF $dst,$src" %}
18518 ins_encode %{
18519 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x00);
18520 %}
18521 ins_pipe( pipe_slow );
18522 %}
18523
18524 instruct ReplF_mem(vec dst, memory mem) %{
18525 predicate(UseAVX > 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
18526 match(Set dst (Replicate (LoadF mem)));
18527 format %{ "replicateF $dst,$mem" %}
18528 ins_encode %{
18529 int vlen_enc = vector_length_encoding(this);
18530 __ vbroadcastss($dst$$XMMRegister, $mem$$Address, vlen_enc);
18531 %}
18532 ins_pipe( pipe_slow );
18533 %}
18534
18535 // Replicate float scalar immediate to be vector by loading from const table.
18536 instruct ReplF_imm(vec dst, immF con) %{
18537 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
18538 match(Set dst (Replicate con));
18539 format %{ "replicateF $dst,$con" %}
18540 ins_encode %{
18541 InternalAddress addr = $constantaddress(vreplicate_imm(T_FLOAT, $con$$constant,
18542 VM_Version::supports_sse3() ? (VM_Version::supports_avx() ? 1 : 2) : 4));
18543 int vlen = Matcher::vector_length_in_bytes(this);
18544 __ load_constant_vector(T_FLOAT, $dst$$XMMRegister, addr, vlen);
18545 %}
18546 ins_pipe( pipe_slow );
18547 %}
18548
18549 instruct ReplF_zero(vec dst, immF0 zero) %{
18550 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
18551 match(Set dst (Replicate zero));
18552 format %{ "replicateF $dst,$zero" %}
18553 ins_encode %{
18554 int vlen_enc = vector_length_encoding(this);
18555 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vldq()) {
18556 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18557 } else {
18558 __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
18559 }
18560 %}
18561 ins_pipe( fpu_reg_reg );
18562 %}
18563
18564 // ====================ReplicateD=======================================
18565
18566 // Replicate double (8 bytes) scalar to be vector
18567 instruct vReplD_reg(vec dst, vlRegD src) %{
18568 predicate(UseSSE >= 3 && Matcher::vector_element_basic_type(n) == T_DOUBLE);
18569 match(Set dst (Replicate src));
18570 format %{ "replicateD $dst,$src" %}
18571 ins_encode %{
18572 uint vlen = Matcher::vector_length(this);
18573 int vlen_enc = vector_length_encoding(this);
18574 if (vlen <= 2) {
18575 __ movddup($dst$$XMMRegister, $src$$XMMRegister);
18576 } else if (VM_Version::supports_avx2()) {
18577 __ vbroadcastsd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc); // reg-to-reg variant requires AVX2
18578 } else {
18579 assert(vlen == 4, "sanity");
18580 __ movddup($dst$$XMMRegister, $src$$XMMRegister);
18581 __ vinsertf128_high($dst$$XMMRegister, $dst$$XMMRegister);
18582 }
18583 %}
18584 ins_pipe( pipe_slow );
18585 %}
18586
18587 instruct ReplD_reg(vec dst, vlRegD src) %{
18588 predicate(UseSSE < 3 && Matcher::vector_element_basic_type(n) == T_DOUBLE);
18589 match(Set dst (Replicate src));
18590 format %{ "replicateD $dst,$src" %}
18591 ins_encode %{
18592 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x44);
18593 %}
18594 ins_pipe( pipe_slow );
18595 %}
18596
18597 instruct ReplD_mem(vec dst, memory mem) %{
18598 predicate(UseSSE >= 3 && Matcher::vector_element_basic_type(n) == T_DOUBLE);
18599 match(Set dst (Replicate (LoadD mem)));
18600 format %{ "replicateD $dst,$mem" %}
18601 ins_encode %{
18602 if (Matcher::vector_length(this) >= 4) {
18603 int vlen_enc = vector_length_encoding(this);
18604 __ vbroadcastsd($dst$$XMMRegister, $mem$$Address, vlen_enc);
18605 } else {
18606 __ movddup($dst$$XMMRegister, $mem$$Address);
18607 }
18608 %}
18609 ins_pipe( pipe_slow );
18610 %}
18611
18612 // Replicate double (8 byte) scalar immediate to be vector by loading from const table.
18613 instruct ReplD_imm(vec dst, immD con) %{
18614 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
18615 match(Set dst (Replicate con));
18616 format %{ "replicateD $dst,$con" %}
18617 ins_encode %{
18618 InternalAddress addr = $constantaddress(vreplicate_imm(T_DOUBLE, $con$$constant, VM_Version::supports_sse3() ? 1 : 2));
18619 int vlen = Matcher::vector_length_in_bytes(this);
18620 __ load_constant_vector(T_DOUBLE, $dst$$XMMRegister, addr, vlen);
18621 %}
18622 ins_pipe( pipe_slow );
18623 %}
18624
18625 instruct ReplD_zero(vec dst, immD0 zero) %{
18626 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
18627 match(Set dst (Replicate zero));
18628 format %{ "replicateD $dst,$zero" %}
18629 ins_encode %{
18630 int vlen_enc = vector_length_encoding(this);
18631 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vldq()) {
18632 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18633 } else {
18634 __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
18635 }
18636 %}
18637 ins_pipe( fpu_reg_reg );
18638 %}
18639
18640 // ====================VECTOR INSERT=======================================
18641
18642 instruct insert(vec dst, rRegI val, immU8 idx) %{
18643 predicate(Matcher::vector_length_in_bytes(n) < 32);
18644 match(Set dst (VectorInsert (Binary dst val) idx));
18645 format %{ "vector_insert $dst,$val,$idx" %}
18646 ins_encode %{
18647 assert(UseSSE >= 4, "required");
18648 assert(Matcher::vector_length_in_bytes(this) >= 8, "required");
18649
18650 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18651
18652 assert(is_integral_type(elem_bt), "");
18653 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18654
18655 __ insert(elem_bt, $dst$$XMMRegister, $val$$Register, $idx$$constant);
18656 %}
18657 ins_pipe( pipe_slow );
18658 %}
18659
18660 instruct insert32(vec dst, vec src, rRegI val, immU8 idx, vec vtmp) %{
18661 predicate(Matcher::vector_length_in_bytes(n) == 32);
18662 match(Set dst (VectorInsert (Binary src val) idx));
18663 effect(TEMP vtmp);
18664 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18665 ins_encode %{
18666 int vlen_enc = Assembler::AVX_256bit;
18667 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18668 int elem_per_lane = 16/type2aelembytes(elem_bt);
18669 int log2epr = log2(elem_per_lane);
18670
18671 assert(is_integral_type(elem_bt), "sanity");
18672 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18673
18674 uint x_idx = $idx$$constant & right_n_bits(log2epr);
18675 uint y_idx = ($idx$$constant >> log2epr) & 1;
18676 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18677 __ vinsert(elem_bt, $vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
18678 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18679 %}
18680 ins_pipe( pipe_slow );
18681 %}
18682
18683 instruct insert64(vec dst, vec src, rRegI val, immU8 idx, legVec vtmp) %{
18684 predicate(Matcher::vector_length_in_bytes(n) == 64);
18685 match(Set dst (VectorInsert (Binary src val) idx));
18686 effect(TEMP vtmp);
18687 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18688 ins_encode %{
18689 assert(UseAVX > 2, "sanity");
18690
18691 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18692 int elem_per_lane = 16/type2aelembytes(elem_bt);
18693 int log2epr = log2(elem_per_lane);
18694
18695 assert(is_integral_type(elem_bt), "");
18696 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18697
18698 uint x_idx = $idx$$constant & right_n_bits(log2epr);
18699 uint y_idx = ($idx$$constant >> log2epr) & 3;
18700 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18701 __ vinsert(elem_bt, $vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
18702 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18703 %}
18704 ins_pipe( pipe_slow );
18705 %}
18706
18707 instruct insert2L(vec dst, rRegL val, immU8 idx) %{
18708 predicate(Matcher::vector_length(n) == 2);
18709 match(Set dst (VectorInsert (Binary dst val) idx));
18710 format %{ "vector_insert $dst,$val,$idx" %}
18711 ins_encode %{
18712 assert(UseSSE >= 4, "required");
18713 assert(Matcher::vector_element_basic_type(this) == T_LONG, "");
18714 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18715
18716 __ pinsrq($dst$$XMMRegister, $val$$Register, $idx$$constant);
18717 %}
18718 ins_pipe( pipe_slow );
18719 %}
18720
18721 instruct insert4L(vec dst, vec src, rRegL val, immU8 idx, vec vtmp) %{
18722 predicate(Matcher::vector_length(n) == 4);
18723 match(Set dst (VectorInsert (Binary src val) idx));
18724 effect(TEMP vtmp);
18725 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18726 ins_encode %{
18727 assert(Matcher::vector_element_basic_type(this) == T_LONG, "");
18728 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18729
18730 uint x_idx = $idx$$constant & right_n_bits(1);
18731 uint y_idx = ($idx$$constant >> 1) & 1;
18732 int vlen_enc = Assembler::AVX_256bit;
18733 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18734 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
18735 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18736 %}
18737 ins_pipe( pipe_slow );
18738 %}
18739
18740 instruct insert8L(vec dst, vec src, rRegL val, immU8 idx, legVec vtmp) %{
18741 predicate(Matcher::vector_length(n) == 8);
18742 match(Set dst (VectorInsert (Binary src val) idx));
18743 effect(TEMP vtmp);
18744 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18745 ins_encode %{
18746 assert(Matcher::vector_element_basic_type(this) == T_LONG, "sanity");
18747 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18748
18749 uint x_idx = $idx$$constant & right_n_bits(1);
18750 uint y_idx = ($idx$$constant >> 1) & 3;
18751 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18752 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
18753 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18754 %}
18755 ins_pipe( pipe_slow );
18756 %}
18757
18758 instruct insertF(vec dst, regF val, immU8 idx) %{
18759 predicate(Matcher::vector_length(n) < 8);
18760 match(Set dst (VectorInsert (Binary dst val) idx));
18761 format %{ "vector_insert $dst,$val,$idx" %}
18762 ins_encode %{
18763 assert(UseSSE >= 4, "sanity");
18764
18765 assert(Matcher::vector_element_basic_type(this) == T_FLOAT, "sanity");
18766 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18767
18768 uint x_idx = $idx$$constant & right_n_bits(2);
18769 __ insertps($dst$$XMMRegister, $val$$XMMRegister, x_idx << 4);
18770 %}
18771 ins_pipe( pipe_slow );
18772 %}
18773
18774 instruct vinsertF(vec dst, vec src, regF val, immU8 idx, vec vtmp) %{
18775 predicate(Matcher::vector_length(n) >= 8);
18776 match(Set dst (VectorInsert (Binary src val) idx));
18777 effect(TEMP vtmp);
18778 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18779 ins_encode %{
18780 assert(Matcher::vector_element_basic_type(this) == T_FLOAT, "sanity");
18781 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18782
18783 int vlen = Matcher::vector_length(this);
18784 uint x_idx = $idx$$constant & right_n_bits(2);
18785 if (vlen == 8) {
18786 uint y_idx = ($idx$$constant >> 2) & 1;
18787 int vlen_enc = Assembler::AVX_256bit;
18788 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18789 __ vinsertps($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$XMMRegister, x_idx << 4);
18790 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18791 } else {
18792 assert(vlen == 16, "sanity");
18793 uint y_idx = ($idx$$constant >> 2) & 3;
18794 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18795 __ vinsertps($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$XMMRegister, x_idx << 4);
18796 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18797 }
18798 %}
18799 ins_pipe( pipe_slow );
18800 %}
18801
18802 instruct insert2D(vec dst, regD val, immU8 idx, rRegL tmp) %{
18803 predicate(Matcher::vector_length(n) == 2);
18804 match(Set dst (VectorInsert (Binary dst val) idx));
18805 effect(TEMP tmp);
18806 format %{ "vector_insert $dst,$val,$idx\t!using $tmp as TEMP" %}
18807 ins_encode %{
18808 assert(UseSSE >= 4, "sanity");
18809 assert(Matcher::vector_element_basic_type(this) == T_DOUBLE, "sanity");
18810 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18811
18812 __ movq($tmp$$Register, $val$$XMMRegister);
18813 __ pinsrq($dst$$XMMRegister, $tmp$$Register, $idx$$constant);
18814 %}
18815 ins_pipe( pipe_slow );
18816 %}
18817
18818 instruct insert4D(vec dst, vec src, regD val, immU8 idx, rRegL tmp, vec vtmp) %{
18819 predicate(Matcher::vector_length(n) == 4);
18820 match(Set dst (VectorInsert (Binary src val) idx));
18821 effect(TEMP vtmp, TEMP tmp);
18822 format %{ "vector_insert $dst,$src,$val,$idx\t!using $tmp, $vtmp as TEMP" %}
18823 ins_encode %{
18824 assert(Matcher::vector_element_basic_type(this) == T_DOUBLE, "sanity");
18825 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18826
18827 uint x_idx = $idx$$constant & right_n_bits(1);
18828 uint y_idx = ($idx$$constant >> 1) & 1;
18829 int vlen_enc = Assembler::AVX_256bit;
18830 __ movq($tmp$$Register, $val$$XMMRegister);
18831 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18832 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $tmp$$Register, x_idx);
18833 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18834 %}
18835 ins_pipe( pipe_slow );
18836 %}
18837
18838 instruct insert8D(vec dst, vec src, regD val, immI idx, rRegL tmp, legVec vtmp) %{
18839 predicate(Matcher::vector_length(n) == 8);
18840 match(Set dst (VectorInsert (Binary src val) idx));
18841 effect(TEMP tmp, TEMP vtmp);
18842 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18843 ins_encode %{
18844 assert(Matcher::vector_element_basic_type(this) == T_DOUBLE, "sanity");
18845 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18846
18847 uint x_idx = $idx$$constant & right_n_bits(1);
18848 uint y_idx = ($idx$$constant >> 1) & 3;
18849 __ movq($tmp$$Register, $val$$XMMRegister);
18850 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18851 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $tmp$$Register, x_idx);
18852 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18853 %}
18854 ins_pipe( pipe_slow );
18855 %}
18856
18857 // ====================REDUCTION ARITHMETIC=======================================
18858
18859 // =======================Int Reduction==========================================
18860
18861 instruct reductionI(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
18862 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_INT); // src2
18863 match(Set dst (AddReductionVI src1 src2));
18864 match(Set dst (MulReductionVI src1 src2));
18865 match(Set dst (AndReductionV src1 src2));
18866 match(Set dst ( OrReductionV src1 src2));
18867 match(Set dst (XorReductionV src1 src2));
18868 match(Set dst (MinReductionV src1 src2));
18869 match(Set dst (MaxReductionV src1 src2));
18870 match(Set dst (UMinReductionV src1 src2));
18871 match(Set dst (UMaxReductionV src1 src2));
18872 effect(TEMP vtmp1, TEMP vtmp2);
18873 format %{ "vector_reduction_int $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
18874 ins_encode %{
18875 int opcode = this->ideal_Opcode();
18876 int vlen = Matcher::vector_length(this, $src2);
18877 __ reduceI(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
18878 %}
18879 ins_pipe( pipe_slow );
18880 %}
18881
18882 // =======================Long Reduction==========================================
18883
18884 instruct reductionL(rRegL dst, rRegL src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
18885 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG && !VM_Version::supports_avx512dq());
18886 match(Set dst (AddReductionVL src1 src2));
18887 match(Set dst (MulReductionVL src1 src2));
18888 match(Set dst (AndReductionV src1 src2));
18889 match(Set dst ( OrReductionV src1 src2));
18890 match(Set dst (XorReductionV src1 src2));
18891 match(Set dst (MinReductionV src1 src2));
18892 match(Set dst (MaxReductionV src1 src2));
18893 match(Set dst (UMinReductionV src1 src2));
18894 match(Set dst (UMaxReductionV src1 src2));
18895 effect(TEMP vtmp1, TEMP vtmp2);
18896 format %{ "vector_reduction_long $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
18897 ins_encode %{
18898 int opcode = this->ideal_Opcode();
18899 int vlen = Matcher::vector_length(this, $src2);
18900 __ reduceL(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
18901 %}
18902 ins_pipe( pipe_slow );
18903 %}
18904
18905 instruct reductionL_avx512dq(rRegL dst, rRegL src1, vec src2, vec vtmp1, vec vtmp2) %{
18906 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG && VM_Version::supports_avx512dq());
18907 match(Set dst (AddReductionVL src1 src2));
18908 match(Set dst (MulReductionVL src1 src2));
18909 match(Set dst (AndReductionV src1 src2));
18910 match(Set dst ( OrReductionV src1 src2));
18911 match(Set dst (XorReductionV src1 src2));
18912 match(Set dst (MinReductionV src1 src2));
18913 match(Set dst (MaxReductionV src1 src2));
18914 match(Set dst (UMinReductionV src1 src2));
18915 match(Set dst (UMaxReductionV src1 src2));
18916 effect(TEMP vtmp1, TEMP vtmp2);
18917 format %{ "vector_reduction_long $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
18918 ins_encode %{
18919 int opcode = this->ideal_Opcode();
18920 int vlen = Matcher::vector_length(this, $src2);
18921 __ reduceL(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
18922 %}
18923 ins_pipe( pipe_slow );
18924 %}
18925
18926 // =======================Float Reduction==========================================
18927
18928 instruct reductionF128(regF dst, vec src, vec vtmp) %{
18929 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) <= 4); // src
18930 match(Set dst (AddReductionVF dst src));
18931 match(Set dst (MulReductionVF dst src));
18932 effect(TEMP dst, TEMP vtmp);
18933 format %{ "vector_reduction_float $dst,$src ; using $vtmp as TEMP" %}
18934 ins_encode %{
18935 int opcode = this->ideal_Opcode();
18936 int vlen = Matcher::vector_length(this, $src);
18937 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister);
18938 %}
18939 ins_pipe( pipe_slow );
18940 %}
18941
18942 instruct reduction8F(regF dst, vec src, vec vtmp1, vec vtmp2) %{
18943 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src
18944 match(Set dst (AddReductionVF dst src));
18945 match(Set dst (MulReductionVF dst src));
18946 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
18947 format %{ "vector_reduction_float $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
18948 ins_encode %{
18949 int opcode = this->ideal_Opcode();
18950 int vlen = Matcher::vector_length(this, $src);
18951 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
18952 %}
18953 ins_pipe( pipe_slow );
18954 %}
18955
18956 instruct reduction16F(regF dst, legVec src, legVec vtmp1, legVec vtmp2) %{
18957 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 16); // src
18958 match(Set dst (AddReductionVF dst src));
18959 match(Set dst (MulReductionVF dst src));
18960 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
18961 format %{ "vector_reduction_float $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
18962 ins_encode %{
18963 int opcode = this->ideal_Opcode();
18964 int vlen = Matcher::vector_length(this, $src);
18965 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
18966 %}
18967 ins_pipe( pipe_slow );
18968 %}
18969
18970
18971 instruct unordered_reduction2F(regF dst, regF src1, vec src2) %{
18972 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
18973 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
18974 // src1 contains reduction identity
18975 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 2); // src2
18976 match(Set dst (AddReductionVF src1 src2));
18977 match(Set dst (MulReductionVF src1 src2));
18978 effect(TEMP dst);
18979 format %{ "vector_reduction_float $dst,$src1,$src2 ;" %}
18980 ins_encode %{
18981 int opcode = this->ideal_Opcode();
18982 int vlen = Matcher::vector_length(this, $src2);
18983 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister);
18984 %}
18985 ins_pipe( pipe_slow );
18986 %}
18987
18988 instruct unordered_reduction4F(regF dst, regF src1, vec src2, vec vtmp) %{
18989 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
18990 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
18991 // src1 contains reduction identity
18992 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 4); // src2
18993 match(Set dst (AddReductionVF src1 src2));
18994 match(Set dst (MulReductionVF src1 src2));
18995 effect(TEMP dst, TEMP vtmp);
18996 format %{ "vector_reduction_float $dst,$src1,$src2 ; using $vtmp as TEMP" %}
18997 ins_encode %{
18998 int opcode = this->ideal_Opcode();
18999 int vlen = Matcher::vector_length(this, $src2);
19000 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp$$XMMRegister);
19001 %}
19002 ins_pipe( pipe_slow );
19003 %}
19004
19005 instruct unordered_reduction8F(regF dst, regF src1, vec src2, vec vtmp1, vec vtmp2) %{
19006 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
19007 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19008 // src1 contains reduction identity
19009 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src2
19010 match(Set dst (AddReductionVF src1 src2));
19011 match(Set dst (MulReductionVF src1 src2));
19012 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19013 format %{ "vector_reduction_float $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19014 ins_encode %{
19015 int opcode = this->ideal_Opcode();
19016 int vlen = Matcher::vector_length(this, $src2);
19017 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19018 %}
19019 ins_pipe( pipe_slow );
19020 %}
19021
19022 instruct unordered_reduction16F(regF dst, regF src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19023 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
19024 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19025 // src1 contains reduction identity
19026 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 16); // src2
19027 match(Set dst (AddReductionVF src1 src2));
19028 match(Set dst (MulReductionVF src1 src2));
19029 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19030 format %{ "vector_reduction_float $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19031 ins_encode %{
19032 int opcode = this->ideal_Opcode();
19033 int vlen = Matcher::vector_length(this, $src2);
19034 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19035 %}
19036 ins_pipe( pipe_slow );
19037 %}
19038
19039 // =======================Double Reduction==========================================
19040
19041 instruct reduction2D(regD dst, vec src, vec vtmp) %{
19042 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 2); // src
19043 match(Set dst (AddReductionVD dst src));
19044 match(Set dst (MulReductionVD dst src));
19045 effect(TEMP dst, TEMP vtmp);
19046 format %{ "vector_reduction_double $dst,$src ; using $vtmp as TEMP" %}
19047 ins_encode %{
19048 int opcode = this->ideal_Opcode();
19049 int vlen = Matcher::vector_length(this, $src);
19050 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister);
19051 %}
19052 ins_pipe( pipe_slow );
19053 %}
19054
19055 instruct reduction4D(regD dst, vec src, vec vtmp1, vec vtmp2) %{
19056 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 4); // src
19057 match(Set dst (AddReductionVD dst src));
19058 match(Set dst (MulReductionVD dst src));
19059 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19060 format %{ "vector_reduction_double $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
19061 ins_encode %{
19062 int opcode = this->ideal_Opcode();
19063 int vlen = Matcher::vector_length(this, $src);
19064 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19065 %}
19066 ins_pipe( pipe_slow );
19067 %}
19068
19069 instruct reduction8D(regD dst, legVec src, legVec vtmp1, legVec vtmp2) %{
19070 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src
19071 match(Set dst (AddReductionVD dst src));
19072 match(Set dst (MulReductionVD dst src));
19073 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19074 format %{ "vector_reduction_double $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
19075 ins_encode %{
19076 int opcode = this->ideal_Opcode();
19077 int vlen = Matcher::vector_length(this, $src);
19078 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19079 %}
19080 ins_pipe( pipe_slow );
19081 %}
19082
19083 instruct unordered_reduction2D(regD dst, regD src1, vec src2) %{
19084 // Non-strictly ordered floating-point add/mul reduction for doubles. This rule is
19085 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19086 // src1 contains reduction identity
19087 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 2); // src2
19088 match(Set dst (AddReductionVD src1 src2));
19089 match(Set dst (MulReductionVD src1 src2));
19090 effect(TEMP dst);
19091 format %{ "vector_reduction_double $dst,$src1,$src2 ;" %}
19092 ins_encode %{
19093 int opcode = this->ideal_Opcode();
19094 int vlen = Matcher::vector_length(this, $src2);
19095 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister);
19096 %}
19097 ins_pipe( pipe_slow );
19098 %}
19099
19100 instruct unordered_reduction4D(regD dst, regD src1, vec src2, vec vtmp) %{
19101 // Non-strictly ordered floating-point add/mul reduction for doubles. This rule is
19102 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19103 // src1 contains reduction identity
19104 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 4); // src2
19105 match(Set dst (AddReductionVD src1 src2));
19106 match(Set dst (MulReductionVD src1 src2));
19107 effect(TEMP dst, TEMP vtmp);
19108 format %{ "vector_reduction_double $dst,$src1,$src2 ; using $vtmp as TEMP" %}
19109 ins_encode %{
19110 int opcode = this->ideal_Opcode();
19111 int vlen = Matcher::vector_length(this, $src2);
19112 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp$$XMMRegister);
19113 %}
19114 ins_pipe( pipe_slow );
19115 %}
19116
19117 instruct unordered_reduction8D(regD dst, regD src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19118 // Non-strictly ordered floating-point add/mul reduction for doubles. This rule is
19119 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19120 // src1 contains reduction identity
19121 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src2
19122 match(Set dst (AddReductionVD src1 src2));
19123 match(Set dst (MulReductionVD src1 src2));
19124 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19125 format %{ "vector_reduction_double $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19126 ins_encode %{
19127 int opcode = this->ideal_Opcode();
19128 int vlen = Matcher::vector_length(this, $src2);
19129 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19130 %}
19131 ins_pipe( pipe_slow );
19132 %}
19133
19134 // =======================Byte Reduction==========================================
19135
19136 instruct reductionB(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19137 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE && !VM_Version::supports_avx512bw());
19138 match(Set dst (AddReductionVI src1 src2));
19139 match(Set dst (AndReductionV src1 src2));
19140 match(Set dst ( OrReductionV src1 src2));
19141 match(Set dst (XorReductionV src1 src2));
19142 match(Set dst (MinReductionV src1 src2));
19143 match(Set dst (MaxReductionV src1 src2));
19144 match(Set dst (UMinReductionV src1 src2));
19145 match(Set dst (UMaxReductionV src1 src2));
19146 effect(TEMP vtmp1, TEMP vtmp2);
19147 format %{ "vector_reduction_byte $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19148 ins_encode %{
19149 int opcode = this->ideal_Opcode();
19150 int vlen = Matcher::vector_length(this, $src2);
19151 __ reduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19152 %}
19153 ins_pipe( pipe_slow );
19154 %}
19155
19156 instruct reductionB_avx512bw(rRegI dst, rRegI src1, vec src2, vec vtmp1, vec vtmp2) %{
19157 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE && VM_Version::supports_avx512bw());
19158 match(Set dst (AddReductionVI src1 src2));
19159 match(Set dst (AndReductionV src1 src2));
19160 match(Set dst ( OrReductionV src1 src2));
19161 match(Set dst (XorReductionV src1 src2));
19162 match(Set dst (MinReductionV src1 src2));
19163 match(Set dst (MaxReductionV src1 src2));
19164 match(Set dst (UMinReductionV src1 src2));
19165 match(Set dst (UMaxReductionV src1 src2));
19166 effect(TEMP vtmp1, TEMP vtmp2);
19167 format %{ "vector_reduction_byte $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19168 ins_encode %{
19169 int opcode = this->ideal_Opcode();
19170 int vlen = Matcher::vector_length(this, $src2);
19171 __ reduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19172 %}
19173 ins_pipe( pipe_slow );
19174 %}
19175
19176 // =======================Short Reduction==========================================
19177
19178 instruct reductionS(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19179 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_SHORT); // src2
19180 match(Set dst (AddReductionVI src1 src2));
19181 match(Set dst (MulReductionVI src1 src2));
19182 match(Set dst (AndReductionV src1 src2));
19183 match(Set dst ( OrReductionV src1 src2));
19184 match(Set dst (XorReductionV src1 src2));
19185 match(Set dst (MinReductionV src1 src2));
19186 match(Set dst (MaxReductionV src1 src2));
19187 match(Set dst (UMinReductionV src1 src2));
19188 match(Set dst (UMaxReductionV src1 src2));
19189 effect(TEMP vtmp1, TEMP vtmp2);
19190 format %{ "vector_reduction_short $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19191 ins_encode %{
19192 int opcode = this->ideal_Opcode();
19193 int vlen = Matcher::vector_length(this, $src2);
19194 __ reduceS(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19195 %}
19196 ins_pipe( pipe_slow );
19197 %}
19198
19199 // =======================Mul Reduction==========================================
19200
19201 instruct mul_reductionB(rRegI dst, rRegI src1, vec src2, vec vtmp1, vec vtmp2) %{
19202 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE &&
19203 Matcher::vector_length(n->in(2)) <= 32); // src2
19204 match(Set dst (MulReductionVI src1 src2));
19205 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19206 format %{ "vector_mul_reduction_byte $dst,$src1,$src2; using $vtmp1, $vtmp2 as TEMP" %}
19207 ins_encode %{
19208 int opcode = this->ideal_Opcode();
19209 int vlen = Matcher::vector_length(this, $src2);
19210 __ mulreduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19211 %}
19212 ins_pipe( pipe_slow );
19213 %}
19214
19215 instruct mul_reduction64B(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19216 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE &&
19217 Matcher::vector_length(n->in(2)) == 64); // src2
19218 match(Set dst (MulReductionVI src1 src2));
19219 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19220 format %{ "vector_mul_reduction_byte $dst,$src1,$src2; using $vtmp1, $vtmp2 as TEMP" %}
19221 ins_encode %{
19222 int opcode = this->ideal_Opcode();
19223 int vlen = Matcher::vector_length(this, $src2);
19224 __ mulreduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19225 %}
19226 ins_pipe( pipe_slow );
19227 %}
19228
19229 //--------------------Min/Max Float Reduction --------------------
19230 // Float Min Reduction
19231 instruct minmax_reduction2F(legRegF dst, immF src1, legVec src2, legVec tmp, legVec atmp,
19232 legVec btmp, legVec xmm_1, rFlagsReg cr) %{
19233 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19234 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19235 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19236 Matcher::vector_length(n->in(2)) == 2);
19237 match(Set dst (MinReductionV src1 src2));
19238 match(Set dst (MaxReductionV src1 src2));
19239 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_1, KILL cr);
19240 format %{ "vector_minmax2F_reduction $dst,$src1,$src2 ; using $tmp, $atmp, $btmp, $xmm_1 as TEMP" %}
19241 ins_encode %{
19242 assert(UseAVX > 0, "sanity");
19243
19244 int opcode = this->ideal_Opcode();
19245 int vlen = Matcher::vector_length(this, $src2);
19246 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, $tmp$$XMMRegister,
19247 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_1$$XMMRegister);
19248 %}
19249 ins_pipe( pipe_slow );
19250 %}
19251
19252 instruct minmax_reductionF(legRegF dst, immF src1, legVec src2, legVec tmp, legVec atmp,
19253 legVec btmp, legVec xmm_0, legVec xmm_1, rFlagsReg cr) %{
19254 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19255 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19256 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19257 Matcher::vector_length(n->in(2)) >= 4);
19258 match(Set dst (MinReductionV src1 src2));
19259 match(Set dst (MaxReductionV src1 src2));
19260 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_0, TEMP xmm_1, KILL cr);
19261 format %{ "vector_minmaxF_reduction $dst,$src1,$src2 ; using $tmp, $atmp, $btmp, $xmm_0, $xmm_1 as TEMP" %}
19262 ins_encode %{
19263 assert(UseAVX > 0, "sanity");
19264
19265 int opcode = this->ideal_Opcode();
19266 int vlen = Matcher::vector_length(this, $src2);
19267 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, $tmp$$XMMRegister,
19268 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_0$$XMMRegister, $xmm_1$$XMMRegister);
19269 %}
19270 ins_pipe( pipe_slow );
19271 %}
19272
19273 instruct minmax_reduction2F_av(legRegF dst, legVec src, legVec tmp, legVec atmp,
19274 legVec btmp, legVec xmm_1, rFlagsReg cr) %{
19275 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19276 Matcher::vector_length(n->in(2)) == 2);
19277 match(Set dst (MinReductionV dst src));
19278 match(Set dst (MaxReductionV dst src));
19279 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_1, KILL cr);
19280 format %{ "vector_minmax2F_reduction $dst,$src ; using $tmp, $atmp, $btmp, $xmm_1 as TEMP" %}
19281 ins_encode %{
19282 assert(UseAVX > 0, "sanity");
19283
19284 int opcode = this->ideal_Opcode();
19285 int vlen = Matcher::vector_length(this, $src);
19286 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, $tmp$$XMMRegister,
19287 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_1$$XMMRegister);
19288 %}
19289 ins_pipe( pipe_slow );
19290 %}
19291
19292
19293 instruct minmax_reductionF_av(legRegF dst, legVec src, legVec tmp, legVec atmp, legVec btmp,
19294 legVec xmm_0, legVec xmm_1, rFlagsReg cr) %{
19295 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19296 Matcher::vector_length(n->in(2)) >= 4);
19297 match(Set dst (MinReductionV dst src));
19298 match(Set dst (MaxReductionV dst src));
19299 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_0, TEMP xmm_1, KILL cr);
19300 format %{ "vector_minmaxF_reduction $dst,$src ; using $tmp, $atmp, $btmp, $xmm_0, $xmm_1 as TEMP" %}
19301 ins_encode %{
19302 assert(UseAVX > 0, "sanity");
19303
19304 int opcode = this->ideal_Opcode();
19305 int vlen = Matcher::vector_length(this, $src);
19306 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, $tmp$$XMMRegister,
19307 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_0$$XMMRegister, $xmm_1$$XMMRegister);
19308 %}
19309 ins_pipe( pipe_slow );
19310 %}
19311
19312 instruct minmax_reduction2F_avx10_2(regF dst, immF src1, vec src2, vec xtmp1) %{
19313 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19314 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19315 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19316 Matcher::vector_length(n->in(2)) == 2);
19317 match(Set dst (MinReductionV src1 src2));
19318 match(Set dst (MaxReductionV src1 src2));
19319 effect(TEMP dst, TEMP xtmp1);
19320 format %{ "vector_minmax_reduction $dst, $src1, $src2 \t; using $xtmp1 as TEMP" %}
19321 ins_encode %{
19322 int opcode = this->ideal_Opcode();
19323 int vlen = Matcher::vector_length(this, $src2);
19324 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister,
19325 xnoreg, xnoreg, xnoreg, $xtmp1$$XMMRegister);
19326 %}
19327 ins_pipe( pipe_slow );
19328 %}
19329
19330 instruct minmax_reductionF_avx10_2(regF dst, immF src1, vec src2, vec xtmp1, vec xtmp2) %{
19331 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19332 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19333 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19334 Matcher::vector_length(n->in(2)) >= 4);
19335 match(Set dst (MinReductionV src1 src2));
19336 match(Set dst (MaxReductionV src1 src2));
19337 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19338 format %{ "vector_minmax_reduction $dst, $src1, $src2 \t; using $xtmp1 and $xtmp2 as TEMP" %}
19339 ins_encode %{
19340 int opcode = this->ideal_Opcode();
19341 int vlen = Matcher::vector_length(this, $src2);
19342 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, xnoreg, xnoreg,
19343 xnoreg, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19344 %}
19345 ins_pipe( pipe_slow );
19346 %}
19347
19348 instruct minmax_reduction2F_av_avx10_2(regF dst, vec src, vec xtmp1) %{
19349 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19350 Matcher::vector_length(n->in(2)) == 2);
19351 match(Set dst (MinReductionV dst src));
19352 match(Set dst (MaxReductionV dst src));
19353 effect(TEMP dst, TEMP xtmp1);
19354 format %{ "vector_minmax2F_reduction $dst, $src \t; using $xtmp1 as TEMP" %}
19355 ins_encode %{
19356 int opcode = this->ideal_Opcode();
19357 int vlen = Matcher::vector_length(this, $src);
19358 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, xnoreg, xnoreg, xnoreg,
19359 $xtmp1$$XMMRegister);
19360 %}
19361 ins_pipe( pipe_slow );
19362 %}
19363
19364 instruct minmax_reductionF_av_avx10_2(regF dst, vec src, vec xtmp1, vec xtmp2) %{
19365 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19366 Matcher::vector_length(n->in(2)) >= 4);
19367 match(Set dst (MinReductionV dst src));
19368 match(Set dst (MaxReductionV dst src));
19369 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19370 format %{ "vector_minmax2F_reduction $dst, $src \t; using $xtmp1 and $xtmp2 as TEMP" %}
19371 ins_encode %{
19372 int opcode = this->ideal_Opcode();
19373 int vlen = Matcher::vector_length(this, $src);
19374 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, xnoreg, xnoreg, xnoreg,
19375 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19376 %}
19377 ins_pipe( pipe_slow );
19378 %}
19379
19380 //--------------------Min Double Reduction --------------------
19381 instruct minmax_reduction2D(legRegD dst, immD src1, legVec src2, legVec tmp1, legVec tmp2,
19382 legVec tmp3, legVec tmp4, rFlagsReg cr) %{
19383 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19384 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19385 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19386 Matcher::vector_length(n->in(2)) == 2);
19387 match(Set dst (MinReductionV src1 src2));
19388 match(Set dst (MaxReductionV src1 src2));
19389 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
19390 format %{ "vector_minmax2D_reduction $dst,$src1,$src2 ; using $tmp1, $tmp2, $tmp3, $tmp4 as TEMP" %}
19391 ins_encode %{
19392 assert(UseAVX > 0, "sanity");
19393
19394 int opcode = this->ideal_Opcode();
19395 int vlen = Matcher::vector_length(this, $src2);
19396 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister,
19397 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister);
19398 %}
19399 ins_pipe( pipe_slow );
19400 %}
19401
19402 instruct minmax_reductionD(legRegD dst, immD src1, legVec src2, legVec tmp1, legVec tmp2,
19403 legVec tmp3, legVec tmp4, legVec tmp5, rFlagsReg cr) %{
19404 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19405 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19406 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19407 Matcher::vector_length(n->in(2)) >= 4);
19408 match(Set dst (MinReductionV src1 src2));
19409 match(Set dst (MaxReductionV src1 src2));
19410 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, KILL cr);
19411 format %{ "vector_minmaxD_reduction $dst,$src1,$src2 ; using $tmp1, $tmp2, $tmp3, $tmp4, $tmp5 as TEMP" %}
19412 ins_encode %{
19413 assert(UseAVX > 0, "sanity");
19414
19415 int opcode = this->ideal_Opcode();
19416 int vlen = Matcher::vector_length(this, $src2);
19417 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister,
19418 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister, $tmp5$$XMMRegister);
19419 %}
19420 ins_pipe( pipe_slow );
19421 %}
19422
19423
19424 instruct minmax_reduction2D_av(legRegD dst, legVec src, legVec tmp1, legVec tmp2,
19425 legVec tmp3, legVec tmp4, rFlagsReg cr) %{
19426 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19427 Matcher::vector_length(n->in(2)) == 2);
19428 match(Set dst (MinReductionV dst src));
19429 match(Set dst (MaxReductionV dst src));
19430 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
19431 format %{ "vector_minmax2D_reduction $dst,$src ; using $tmp1, $tmp2, $tmp3, $tmp4 as TEMP" %}
19432 ins_encode %{
19433 assert(UseAVX > 0, "sanity");
19434
19435 int opcode = this->ideal_Opcode();
19436 int vlen = Matcher::vector_length(this, $src);
19437 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19438 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister);
19439 %}
19440 ins_pipe( pipe_slow );
19441 %}
19442
19443 instruct minmax_reductionD_av(legRegD dst, legVec src, legVec tmp1, legVec tmp2, legVec tmp3,
19444 legVec tmp4, legVec tmp5, rFlagsReg cr) %{
19445 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19446 Matcher::vector_length(n->in(2)) >= 4);
19447 match(Set dst (MinReductionV dst src));
19448 match(Set dst (MaxReductionV dst src));
19449 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, KILL cr);
19450 format %{ "vector_minmaxD_reduction $dst,$src ; using $tmp1, $tmp2, $tmp3, $tmp4, $tmp5 as TEMP" %}
19451 ins_encode %{
19452 assert(UseAVX > 0, "sanity");
19453
19454 int opcode = this->ideal_Opcode();
19455 int vlen = Matcher::vector_length(this, $src);
19456 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19457 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister, $tmp5$$XMMRegister);
19458 %}
19459 ins_pipe( pipe_slow );
19460 %}
19461
19462 instruct minmax_reduction2D_avx10_2(regD dst, immD src1, vec src2, vec xtmp1) %{
19463 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19464 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19465 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19466 Matcher::vector_length(n->in(2)) == 2);
19467 match(Set dst (MinReductionV src1 src2));
19468 match(Set dst (MaxReductionV src1 src2));
19469 effect(TEMP dst, TEMP xtmp1);
19470 format %{ "vector_minmax2D_reduction $dst, $src1, $src2 ; using $xtmp1 as TEMP" %}
19471 ins_encode %{
19472 int opcode = this->ideal_Opcode();
19473 int vlen = Matcher::vector_length(this, $src2);
19474 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, xnoreg,
19475 xnoreg, xnoreg, $xtmp1$$XMMRegister);
19476 %}
19477 ins_pipe( pipe_slow );
19478 %}
19479
19480 instruct minmax_reductionD_avx10_2(regD dst, immD src1, vec src2, vec xtmp1, vec xtmp2) %{
19481 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19482 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19483 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19484 Matcher::vector_length(n->in(2)) >= 4);
19485 match(Set dst (MinReductionV src1 src2));
19486 match(Set dst (MaxReductionV src1 src2));
19487 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19488 format %{ "vector_minmaxD_reduction $dst, $src1, $src2 ; using $xtmp1 and $xtmp2 as TEMP" %}
19489 ins_encode %{
19490 int opcode = this->ideal_Opcode();
19491 int vlen = Matcher::vector_length(this, $src2);
19492 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, xnoreg, xnoreg,
19493 xnoreg, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19494 %}
19495 ins_pipe( pipe_slow );
19496 %}
19497
19498
19499 instruct minmax_reduction2D_av_avx10_2(regD dst, vec src, vec xtmp1) %{
19500 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19501 Matcher::vector_length(n->in(2)) == 2);
19502 match(Set dst (MinReductionV dst src));
19503 match(Set dst (MaxReductionV dst src));
19504 effect(TEMP dst, TEMP xtmp1);
19505 format %{ "vector_minmax2D_reduction $dst, $src ; using $xtmp1 as TEMP" %}
19506 ins_encode %{
19507 int opcode = this->ideal_Opcode();
19508 int vlen = Matcher::vector_length(this, $src);
19509 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19510 xnoreg, xnoreg, xnoreg, $xtmp1$$XMMRegister);
19511 %}
19512 ins_pipe( pipe_slow );
19513 %}
19514
19515 instruct minmax_reductionD_av_avx10_2(regD dst, vec src, vec xtmp1, vec xtmp2) %{
19516 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19517 Matcher::vector_length(n->in(2)) >= 4);
19518 match(Set dst (MinReductionV dst src));
19519 match(Set dst (MaxReductionV dst src));
19520 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19521 format %{ "vector_minmaxD_reduction $dst, $src ; using $xtmp1 and $xtmp2 as TEMP" %}
19522 ins_encode %{
19523 int opcode = this->ideal_Opcode();
19524 int vlen = Matcher::vector_length(this, $src);
19525 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19526 xnoreg, xnoreg, xnoreg, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19527 %}
19528 ins_pipe( pipe_slow );
19529 %}
19530
19531 // ====================VECTOR ARITHMETIC=======================================
19532
19533 // --------------------------------- ADD --------------------------------------
19534
19535 // Bytes vector add
19536 instruct vaddB(vec dst, vec src) %{
19537 predicate(UseAVX == 0);
19538 match(Set dst (AddVB dst src));
19539 format %{ "paddb $dst,$src\t! add packedB" %}
19540 ins_encode %{
19541 __ paddb($dst$$XMMRegister, $src$$XMMRegister);
19542 %}
19543 ins_pipe( pipe_slow );
19544 %}
19545
19546 instruct vaddB_reg(vec dst, vec src1, vec src2) %{
19547 predicate(UseAVX > 0);
19548 match(Set dst (AddVB src1 src2));
19549 format %{ "vpaddb $dst,$src1,$src2\t! add packedB" %}
19550 ins_encode %{
19551 int vlen_enc = vector_length_encoding(this);
19552 __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19553 %}
19554 ins_pipe( pipe_slow );
19555 %}
19556
19557 instruct vaddB_mem(vec dst, vec src, memory mem) %{
19558 predicate((UseAVX > 0) &&
19559 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19560 match(Set dst (AddVB src (LoadVector mem)));
19561 format %{ "vpaddb $dst,$src,$mem\t! add packedB" %}
19562 ins_encode %{
19563 int vlen_enc = vector_length_encoding(this);
19564 __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19565 %}
19566 ins_pipe( pipe_slow );
19567 %}
19568
19569 // Shorts/Chars vector add
19570 instruct vaddS(vec dst, vec src) %{
19571 predicate(UseAVX == 0);
19572 match(Set dst (AddVS dst src));
19573 format %{ "paddw $dst,$src\t! add packedS" %}
19574 ins_encode %{
19575 __ paddw($dst$$XMMRegister, $src$$XMMRegister);
19576 %}
19577 ins_pipe( pipe_slow );
19578 %}
19579
19580 instruct vaddS_reg(vec dst, vec src1, vec src2) %{
19581 predicate(UseAVX > 0);
19582 match(Set dst (AddVS src1 src2));
19583 format %{ "vpaddw $dst,$src1,$src2\t! add packedS" %}
19584 ins_encode %{
19585 int vlen_enc = vector_length_encoding(this);
19586 __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19587 %}
19588 ins_pipe( pipe_slow );
19589 %}
19590
19591 instruct vaddS_mem(vec dst, vec src, memory mem) %{
19592 predicate((UseAVX > 0) &&
19593 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19594 match(Set dst (AddVS src (LoadVector mem)));
19595 format %{ "vpaddw $dst,$src,$mem\t! add packedS" %}
19596 ins_encode %{
19597 int vlen_enc = vector_length_encoding(this);
19598 __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19599 %}
19600 ins_pipe( pipe_slow );
19601 %}
19602
19603 // Integers vector add
19604 instruct vaddI(vec dst, vec src) %{
19605 predicate(UseAVX == 0);
19606 match(Set dst (AddVI dst src));
19607 format %{ "paddd $dst,$src\t! add packedI" %}
19608 ins_encode %{
19609 __ paddd($dst$$XMMRegister, $src$$XMMRegister);
19610 %}
19611 ins_pipe( pipe_slow );
19612 %}
19613
19614 instruct vaddI_reg(vec dst, vec src1, vec src2) %{
19615 predicate(UseAVX > 0);
19616 match(Set dst (AddVI src1 src2));
19617 format %{ "vpaddd $dst,$src1,$src2\t! add packedI" %}
19618 ins_encode %{
19619 int vlen_enc = vector_length_encoding(this);
19620 __ vpaddd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19621 %}
19622 ins_pipe( pipe_slow );
19623 %}
19624
19625
19626 instruct vaddI_mem(vec dst, vec src, memory mem) %{
19627 predicate((UseAVX > 0) &&
19628 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19629 match(Set dst (AddVI src (LoadVector mem)));
19630 format %{ "vpaddd $dst,$src,$mem\t! add packedI" %}
19631 ins_encode %{
19632 int vlen_enc = vector_length_encoding(this);
19633 __ vpaddd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19634 %}
19635 ins_pipe( pipe_slow );
19636 %}
19637
19638 // Longs vector add
19639 instruct vaddL(vec dst, vec src) %{
19640 predicate(UseAVX == 0);
19641 match(Set dst (AddVL dst src));
19642 format %{ "paddq $dst,$src\t! add packedL" %}
19643 ins_encode %{
19644 __ paddq($dst$$XMMRegister, $src$$XMMRegister);
19645 %}
19646 ins_pipe( pipe_slow );
19647 %}
19648
19649 instruct vaddL_reg(vec dst, vec src1, vec src2) %{
19650 predicate(UseAVX > 0);
19651 match(Set dst (AddVL src1 src2));
19652 format %{ "vpaddq $dst,$src1,$src2\t! add packedL" %}
19653 ins_encode %{
19654 int vlen_enc = vector_length_encoding(this);
19655 __ vpaddq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19656 %}
19657 ins_pipe( pipe_slow );
19658 %}
19659
19660 instruct vaddL_mem(vec dst, vec src, memory mem) %{
19661 predicate((UseAVX > 0) &&
19662 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19663 match(Set dst (AddVL src (LoadVector mem)));
19664 format %{ "vpaddq $dst,$src,$mem\t! add packedL" %}
19665 ins_encode %{
19666 int vlen_enc = vector_length_encoding(this);
19667 __ vpaddq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19668 %}
19669 ins_pipe( pipe_slow );
19670 %}
19671
19672 // Floats vector add
19673 instruct vaddF(vec dst, vec src) %{
19674 predicate(UseAVX == 0);
19675 match(Set dst (AddVF dst src));
19676 format %{ "addps $dst,$src\t! add packedF" %}
19677 ins_encode %{
19678 __ addps($dst$$XMMRegister, $src$$XMMRegister);
19679 %}
19680 ins_pipe( pipe_slow );
19681 %}
19682
19683 instruct vaddF_reg(vec dst, vec src1, vec src2) %{
19684 predicate(UseAVX > 0);
19685 match(Set dst (AddVF src1 src2));
19686 format %{ "vaddps $dst,$src1,$src2\t! add packedF" %}
19687 ins_encode %{
19688 int vlen_enc = vector_length_encoding(this);
19689 __ vaddps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19690 %}
19691 ins_pipe( pipe_slow );
19692 %}
19693
19694 instruct vaddF_mem(vec dst, vec src, memory mem) %{
19695 predicate((UseAVX > 0) &&
19696 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19697 match(Set dst (AddVF src (LoadVector mem)));
19698 format %{ "vaddps $dst,$src,$mem\t! add packedF" %}
19699 ins_encode %{
19700 int vlen_enc = vector_length_encoding(this);
19701 __ vaddps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19702 %}
19703 ins_pipe( pipe_slow );
19704 %}
19705
19706 // Doubles vector add
19707 instruct vaddD(vec dst, vec src) %{
19708 predicate(UseAVX == 0);
19709 match(Set dst (AddVD dst src));
19710 format %{ "addpd $dst,$src\t! add packedD" %}
19711 ins_encode %{
19712 __ addpd($dst$$XMMRegister, $src$$XMMRegister);
19713 %}
19714 ins_pipe( pipe_slow );
19715 %}
19716
19717 instruct vaddD_reg(vec dst, vec src1, vec src2) %{
19718 predicate(UseAVX > 0);
19719 match(Set dst (AddVD src1 src2));
19720 format %{ "vaddpd $dst,$src1,$src2\t! add packedD" %}
19721 ins_encode %{
19722 int vlen_enc = vector_length_encoding(this);
19723 __ vaddpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19724 %}
19725 ins_pipe( pipe_slow );
19726 %}
19727
19728 instruct vaddD_mem(vec dst, vec src, memory mem) %{
19729 predicate((UseAVX > 0) &&
19730 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19731 match(Set dst (AddVD src (LoadVector mem)));
19732 format %{ "vaddpd $dst,$src,$mem\t! add packedD" %}
19733 ins_encode %{
19734 int vlen_enc = vector_length_encoding(this);
19735 __ vaddpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19736 %}
19737 ins_pipe( pipe_slow );
19738 %}
19739
19740 // --------------------------------- SUB --------------------------------------
19741
19742 // Bytes vector sub
19743 instruct vsubB(vec dst, vec src) %{
19744 predicate(UseAVX == 0);
19745 match(Set dst (SubVB dst src));
19746 format %{ "psubb $dst,$src\t! sub packedB" %}
19747 ins_encode %{
19748 __ psubb($dst$$XMMRegister, $src$$XMMRegister);
19749 %}
19750 ins_pipe( pipe_slow );
19751 %}
19752
19753 instruct vsubB_reg(vec dst, vec src1, vec src2) %{
19754 predicate(UseAVX > 0);
19755 match(Set dst (SubVB src1 src2));
19756 format %{ "vpsubb $dst,$src1,$src2\t! sub packedB" %}
19757 ins_encode %{
19758 int vlen_enc = vector_length_encoding(this);
19759 __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19760 %}
19761 ins_pipe( pipe_slow );
19762 %}
19763
19764 instruct vsubB_mem(vec dst, vec src, memory mem) %{
19765 predicate((UseAVX > 0) &&
19766 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19767 match(Set dst (SubVB src (LoadVector mem)));
19768 format %{ "vpsubb $dst,$src,$mem\t! sub packedB" %}
19769 ins_encode %{
19770 int vlen_enc = vector_length_encoding(this);
19771 __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19772 %}
19773 ins_pipe( pipe_slow );
19774 %}
19775
19776 // Shorts/Chars vector sub
19777 instruct vsubS(vec dst, vec src) %{
19778 predicate(UseAVX == 0);
19779 match(Set dst (SubVS dst src));
19780 format %{ "psubw $dst,$src\t! sub packedS" %}
19781 ins_encode %{
19782 __ psubw($dst$$XMMRegister, $src$$XMMRegister);
19783 %}
19784 ins_pipe( pipe_slow );
19785 %}
19786
19787
19788 instruct vsubS_reg(vec dst, vec src1, vec src2) %{
19789 predicate(UseAVX > 0);
19790 match(Set dst (SubVS src1 src2));
19791 format %{ "vpsubw $dst,$src1,$src2\t! sub packedS" %}
19792 ins_encode %{
19793 int vlen_enc = vector_length_encoding(this);
19794 __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19795 %}
19796 ins_pipe( pipe_slow );
19797 %}
19798
19799 instruct vsubS_mem(vec dst, vec src, memory mem) %{
19800 predicate((UseAVX > 0) &&
19801 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19802 match(Set dst (SubVS src (LoadVector mem)));
19803 format %{ "vpsubw $dst,$src,$mem\t! sub packedS" %}
19804 ins_encode %{
19805 int vlen_enc = vector_length_encoding(this);
19806 __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19807 %}
19808 ins_pipe( pipe_slow );
19809 %}
19810
19811 // Integers vector sub
19812 instruct vsubI(vec dst, vec src) %{
19813 predicate(UseAVX == 0);
19814 match(Set dst (SubVI dst src));
19815 format %{ "psubd $dst,$src\t! sub packedI" %}
19816 ins_encode %{
19817 __ psubd($dst$$XMMRegister, $src$$XMMRegister);
19818 %}
19819 ins_pipe( pipe_slow );
19820 %}
19821
19822 instruct vsubI_reg(vec dst, vec src1, vec src2) %{
19823 predicate(UseAVX > 0);
19824 match(Set dst (SubVI src1 src2));
19825 format %{ "vpsubd $dst,$src1,$src2\t! sub packedI" %}
19826 ins_encode %{
19827 int vlen_enc = vector_length_encoding(this);
19828 __ vpsubd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19829 %}
19830 ins_pipe( pipe_slow );
19831 %}
19832
19833 instruct vsubI_mem(vec dst, vec src, memory mem) %{
19834 predicate((UseAVX > 0) &&
19835 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19836 match(Set dst (SubVI src (LoadVector mem)));
19837 format %{ "vpsubd $dst,$src,$mem\t! sub packedI" %}
19838 ins_encode %{
19839 int vlen_enc = vector_length_encoding(this);
19840 __ vpsubd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19841 %}
19842 ins_pipe( pipe_slow );
19843 %}
19844
19845 // Longs vector sub
19846 instruct vsubL(vec dst, vec src) %{
19847 predicate(UseAVX == 0);
19848 match(Set dst (SubVL dst src));
19849 format %{ "psubq $dst,$src\t! sub packedL" %}
19850 ins_encode %{
19851 __ psubq($dst$$XMMRegister, $src$$XMMRegister);
19852 %}
19853 ins_pipe( pipe_slow );
19854 %}
19855
19856 instruct vsubL_reg(vec dst, vec src1, vec src2) %{
19857 predicate(UseAVX > 0);
19858 match(Set dst (SubVL src1 src2));
19859 format %{ "vpsubq $dst,$src1,$src2\t! sub packedL" %}
19860 ins_encode %{
19861 int vlen_enc = vector_length_encoding(this);
19862 __ vpsubq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19863 %}
19864 ins_pipe( pipe_slow );
19865 %}
19866
19867
19868 instruct vsubL_mem(vec dst, vec src, memory mem) %{
19869 predicate((UseAVX > 0) &&
19870 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19871 match(Set dst (SubVL src (LoadVector mem)));
19872 format %{ "vpsubq $dst,$src,$mem\t! sub packedL" %}
19873 ins_encode %{
19874 int vlen_enc = vector_length_encoding(this);
19875 __ vpsubq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19876 %}
19877 ins_pipe( pipe_slow );
19878 %}
19879
19880 // Floats vector sub
19881 instruct vsubF(vec dst, vec src) %{
19882 predicate(UseAVX == 0);
19883 match(Set dst (SubVF dst src));
19884 format %{ "subps $dst,$src\t! sub packedF" %}
19885 ins_encode %{
19886 __ subps($dst$$XMMRegister, $src$$XMMRegister);
19887 %}
19888 ins_pipe( pipe_slow );
19889 %}
19890
19891 instruct vsubF_reg(vec dst, vec src1, vec src2) %{
19892 predicate(UseAVX > 0);
19893 match(Set dst (SubVF src1 src2));
19894 format %{ "vsubps $dst,$src1,$src2\t! sub packedF" %}
19895 ins_encode %{
19896 int vlen_enc = vector_length_encoding(this);
19897 __ vsubps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19898 %}
19899 ins_pipe( pipe_slow );
19900 %}
19901
19902 instruct vsubF_mem(vec dst, vec src, memory mem) %{
19903 predicate((UseAVX > 0) &&
19904 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19905 match(Set dst (SubVF src (LoadVector mem)));
19906 format %{ "vsubps $dst,$src,$mem\t! sub packedF" %}
19907 ins_encode %{
19908 int vlen_enc = vector_length_encoding(this);
19909 __ vsubps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19910 %}
19911 ins_pipe( pipe_slow );
19912 %}
19913
19914 // Doubles vector sub
19915 instruct vsubD(vec dst, vec src) %{
19916 predicate(UseAVX == 0);
19917 match(Set dst (SubVD dst src));
19918 format %{ "subpd $dst,$src\t! sub packedD" %}
19919 ins_encode %{
19920 __ subpd($dst$$XMMRegister, $src$$XMMRegister);
19921 %}
19922 ins_pipe( pipe_slow );
19923 %}
19924
19925 instruct vsubD_reg(vec dst, vec src1, vec src2) %{
19926 predicate(UseAVX > 0);
19927 match(Set dst (SubVD src1 src2));
19928 format %{ "vsubpd $dst,$src1,$src2\t! sub packedD" %}
19929 ins_encode %{
19930 int vlen_enc = vector_length_encoding(this);
19931 __ vsubpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19932 %}
19933 ins_pipe( pipe_slow );
19934 %}
19935
19936 instruct vsubD_mem(vec dst, vec src, memory mem) %{
19937 predicate((UseAVX > 0) &&
19938 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19939 match(Set dst (SubVD src (LoadVector mem)));
19940 format %{ "vsubpd $dst,$src,$mem\t! sub packedD" %}
19941 ins_encode %{
19942 int vlen_enc = vector_length_encoding(this);
19943 __ vsubpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19944 %}
19945 ins_pipe( pipe_slow );
19946 %}
19947
19948 // --------------------------------- MUL --------------------------------------
19949
19950 // Byte vector mul
19951 instruct vmul8B(vec dst, vec src1, vec src2, vec xtmp) %{
19952 predicate(Matcher::vector_length_in_bytes(n) <= 8);
19953 match(Set dst (MulVB src1 src2));
19954 effect(TEMP dst, TEMP xtmp);
19955 format %{ "mulVB $dst, $src1, $src2\t! using $xtmp as TEMP" %}
19956 ins_encode %{
19957 assert(UseSSE > 3, "required");
19958 __ pmovsxbw($dst$$XMMRegister, $src1$$XMMRegister);
19959 __ pmovsxbw($xtmp$$XMMRegister, $src2$$XMMRegister);
19960 __ pmullw($dst$$XMMRegister, $xtmp$$XMMRegister);
19961 __ psllw($dst$$XMMRegister, 8);
19962 __ psrlw($dst$$XMMRegister, 8);
19963 __ packuswb($dst$$XMMRegister, $dst$$XMMRegister);
19964 %}
19965 ins_pipe( pipe_slow );
19966 %}
19967
19968 instruct vmulB(vec dst, vec src1, vec src2, vec xtmp) %{
19969 predicate(UseAVX == 0 && Matcher::vector_length_in_bytes(n) > 8);
19970 match(Set dst (MulVB src1 src2));
19971 effect(TEMP dst, TEMP xtmp);
19972 format %{ "mulVB $dst, $src1, $src2\t! using $xtmp as TEMP" %}
19973 ins_encode %{
19974 assert(UseSSE > 3, "required");
19975 // Odd-index elements
19976 __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
19977 __ psrlw($dst$$XMMRegister, 8);
19978 __ movdqu($xtmp$$XMMRegister, $src2$$XMMRegister);
19979 __ psrlw($xtmp$$XMMRegister, 8);
19980 __ pmullw($dst$$XMMRegister, $xtmp$$XMMRegister);
19981 __ psllw($dst$$XMMRegister, 8);
19982 // Even-index elements
19983 __ movdqu($xtmp$$XMMRegister, $src1$$XMMRegister);
19984 __ pmullw($xtmp$$XMMRegister, $src2$$XMMRegister);
19985 __ psllw($xtmp$$XMMRegister, 8);
19986 __ psrlw($xtmp$$XMMRegister, 8);
19987 // Combine
19988 __ por($dst$$XMMRegister, $xtmp$$XMMRegister);
19989 %}
19990 ins_pipe( pipe_slow );
19991 %}
19992
19993 instruct vmulB_reg(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2) %{
19994 predicate(UseAVX > 0 && Matcher::vector_length_in_bytes(n) > 8);
19995 match(Set dst (MulVB src1 src2));
19996 effect(TEMP xtmp1, TEMP xtmp2);
19997 format %{ "vmulVB $dst, $src1, $src2\t! using $xtmp1, $xtmp2 as TEMP" %}
19998 ins_encode %{
19999 int vlen_enc = vector_length_encoding(this);
20000 // Odd-index elements
20001 __ vpsrlw($xtmp2$$XMMRegister, $src1$$XMMRegister, 8, vlen_enc);
20002 __ vpsrlw($xtmp1$$XMMRegister, $src2$$XMMRegister, 8, vlen_enc);
20003 __ vpmullw($xtmp2$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20004 __ vpsllw($xtmp2$$XMMRegister, $xtmp2$$XMMRegister, 8, vlen_enc);
20005 // Even-index elements
20006 __ vpmullw($xtmp1$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20007 __ vpsllw($xtmp1$$XMMRegister, $xtmp1$$XMMRegister, 8, vlen_enc);
20008 __ vpsrlw($xtmp1$$XMMRegister, $xtmp1$$XMMRegister, 8, vlen_enc);
20009 // Combine
20010 __ vpor($dst$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20011 %}
20012 ins_pipe( pipe_slow );
20013 %}
20014
20015 // Shorts/Chars vector mul
20016 instruct vmulS(vec dst, vec src) %{
20017 predicate(UseAVX == 0);
20018 match(Set dst (MulVS dst src));
20019 format %{ "pmullw $dst,$src\t! mul packedS" %}
20020 ins_encode %{
20021 __ pmullw($dst$$XMMRegister, $src$$XMMRegister);
20022 %}
20023 ins_pipe( pipe_slow );
20024 %}
20025
20026 instruct vmulS_reg(vec dst, vec src1, vec src2) %{
20027 predicate(UseAVX > 0);
20028 match(Set dst (MulVS src1 src2));
20029 format %{ "vpmullw $dst,$src1,$src2\t! mul packedS" %}
20030 ins_encode %{
20031 int vlen_enc = vector_length_encoding(this);
20032 __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20033 %}
20034 ins_pipe( pipe_slow );
20035 %}
20036
20037 instruct vmulS_mem(vec dst, vec src, memory mem) %{
20038 predicate((UseAVX > 0) &&
20039 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20040 match(Set dst (MulVS src (LoadVector mem)));
20041 format %{ "vpmullw $dst,$src,$mem\t! mul packedS" %}
20042 ins_encode %{
20043 int vlen_enc = vector_length_encoding(this);
20044 __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20045 %}
20046 ins_pipe( pipe_slow );
20047 %}
20048
20049 // Integers vector mul
20050 instruct vmulI(vec dst, vec src) %{
20051 predicate(UseAVX == 0);
20052 match(Set dst (MulVI dst src));
20053 format %{ "pmulld $dst,$src\t! mul packedI" %}
20054 ins_encode %{
20055 assert(UseSSE > 3, "required");
20056 __ pmulld($dst$$XMMRegister, $src$$XMMRegister);
20057 %}
20058 ins_pipe( pipe_slow );
20059 %}
20060
20061 instruct vmulI_reg(vec dst, vec src1, vec src2) %{
20062 predicate(UseAVX > 0);
20063 match(Set dst (MulVI src1 src2));
20064 format %{ "vpmulld $dst,$src1,$src2\t! mul packedI" %}
20065 ins_encode %{
20066 int vlen_enc = vector_length_encoding(this);
20067 __ vpmulld($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20068 %}
20069 ins_pipe( pipe_slow );
20070 %}
20071
20072 instruct vmulI_mem(vec dst, vec src, memory mem) %{
20073 predicate((UseAVX > 0) &&
20074 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20075 match(Set dst (MulVI src (LoadVector mem)));
20076 format %{ "vpmulld $dst,$src,$mem\t! mul packedI" %}
20077 ins_encode %{
20078 int vlen_enc = vector_length_encoding(this);
20079 __ vpmulld($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20080 %}
20081 ins_pipe( pipe_slow );
20082 %}
20083
20084 // Longs vector mul
20085 instruct evmulL_reg(vec dst, vec src1, vec src2) %{
20086 predicate((Matcher::vector_length_in_bytes(n) == 64 &&
20087 VM_Version::supports_avx512dq()) ||
20088 VM_Version::supports_avx512vldq());
20089 match(Set dst (MulVL src1 src2));
20090 ins_cost(500);
20091 format %{ "evpmullq $dst,$src1,$src2\t! mul packedL" %}
20092 ins_encode %{
20093 assert(UseAVX > 2, "required");
20094 int vlen_enc = vector_length_encoding(this);
20095 __ evpmullq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20096 %}
20097 ins_pipe( pipe_slow );
20098 %}
20099
20100 instruct evmulL_mem(vec dst, vec src, memory mem) %{
20101 predicate((Matcher::vector_length_in_bytes(n) == 64 &&
20102 VM_Version::supports_avx512dq()) ||
20103 (Matcher::vector_length_in_bytes(n) > 8 &&
20104 VM_Version::supports_avx512vldq()));
20105 match(Set dst (MulVL src (LoadVector mem)));
20106 format %{ "evpmullq $dst,$src,$mem\t! mul packedL" %}
20107 ins_cost(500);
20108 ins_encode %{
20109 assert(UseAVX > 2, "required");
20110 int vlen_enc = vector_length_encoding(this);
20111 __ evpmullq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20112 %}
20113 ins_pipe( pipe_slow );
20114 %}
20115
20116 instruct vmulL(vec dst, vec src1, vec src2, vec xtmp) %{
20117 predicate(UseAVX == 0);
20118 match(Set dst (MulVL src1 src2));
20119 ins_cost(500);
20120 effect(TEMP dst, TEMP xtmp);
20121 format %{ "mulVL $dst, $src1, $src2\t! using $xtmp as TEMP" %}
20122 ins_encode %{
20123 assert(VM_Version::supports_sse4_1(), "required");
20124 // Get the lo-hi products, only the lower 32 bits is in concerns
20125 __ pshufd($xtmp$$XMMRegister, $src2$$XMMRegister, 0xB1);
20126 __ pmulld($xtmp$$XMMRegister, $src1$$XMMRegister);
20127 __ pshufd($dst$$XMMRegister, $xtmp$$XMMRegister, 0xB1);
20128 __ paddd($dst$$XMMRegister, $xtmp$$XMMRegister);
20129 __ psllq($dst$$XMMRegister, 32);
20130 // Get the lo-lo products
20131 __ movdqu($xtmp$$XMMRegister, $src1$$XMMRegister);
20132 __ pmuludq($xtmp$$XMMRegister, $src2$$XMMRegister);
20133 __ paddq($dst$$XMMRegister, $xtmp$$XMMRegister);
20134 %}
20135 ins_pipe( pipe_slow );
20136 %}
20137
20138 instruct vmulL_reg(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2) %{
20139 predicate(UseAVX > 0 &&
20140 ((Matcher::vector_length_in_bytes(n) == 64 &&
20141 !VM_Version::supports_avx512dq()) ||
20142 (Matcher::vector_length_in_bytes(n) < 64 &&
20143 !VM_Version::supports_avx512vldq())));
20144 match(Set dst (MulVL src1 src2));
20145 effect(TEMP xtmp1, TEMP xtmp2);
20146 ins_cost(500);
20147 format %{ "vmulVL $dst, $src1, $src2\t! using $xtmp1, $xtmp2 as TEMP" %}
20148 ins_encode %{
20149 int vlen_enc = vector_length_encoding(this);
20150 // Get the lo-hi products, only the lower 32 bits is in concerns
20151 __ vpshufd($xtmp1$$XMMRegister, $src2$$XMMRegister, 0xB1, vlen_enc);
20152 __ vpmulld($xtmp1$$XMMRegister, $src1$$XMMRegister, $xtmp1$$XMMRegister, vlen_enc);
20153 __ vpshufd($xtmp2$$XMMRegister, $xtmp1$$XMMRegister, 0xB1, vlen_enc);
20154 __ vpaddd($xtmp2$$XMMRegister, $xtmp2$$XMMRegister, $xtmp1$$XMMRegister, vlen_enc);
20155 __ vpsllq($xtmp2$$XMMRegister, $xtmp2$$XMMRegister, 32, vlen_enc);
20156 // Get the lo-lo products
20157 __ vpmuludq($xtmp1$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20158 __ vpaddq($dst$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20159 %}
20160 ins_pipe( pipe_slow );
20161 %}
20162
20163 instruct vmuludq_reg(vec dst, vec src1, vec src2) %{
20164 predicate(UseAVX > 0 && n->as_MulVL()->has_uint_inputs());
20165 match(Set dst (MulVL src1 src2));
20166 ins_cost(100);
20167 format %{ "vpmuludq $dst,$src1,$src2\t! muludq packedL" %}
20168 ins_encode %{
20169 int vlen_enc = vector_length_encoding(this);
20170 __ vpmuludq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20171 %}
20172 ins_pipe( pipe_slow );
20173 %}
20174
20175 instruct vmuldq_reg(vec dst, vec src1, vec src2) %{
20176 predicate(UseAVX > 0 && n->as_MulVL()->has_int_inputs());
20177 match(Set dst (MulVL src1 src2));
20178 ins_cost(100);
20179 format %{ "vpmuldq $dst,$src1,$src2\t! muldq packedL" %}
20180 ins_encode %{
20181 int vlen_enc = vector_length_encoding(this);
20182 __ vpmuldq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20183 %}
20184 ins_pipe( pipe_slow );
20185 %}
20186
20187 // Floats vector mul
20188 instruct vmulF(vec dst, vec src) %{
20189 predicate(UseAVX == 0);
20190 match(Set dst (MulVF dst src));
20191 format %{ "mulps $dst,$src\t! mul packedF" %}
20192 ins_encode %{
20193 __ mulps($dst$$XMMRegister, $src$$XMMRegister);
20194 %}
20195 ins_pipe( pipe_slow );
20196 %}
20197
20198 instruct vmulF_reg(vec dst, vec src1, vec src2) %{
20199 predicate(UseAVX > 0);
20200 match(Set dst (MulVF src1 src2));
20201 format %{ "vmulps $dst,$src1,$src2\t! mul packedF" %}
20202 ins_encode %{
20203 int vlen_enc = vector_length_encoding(this);
20204 __ vmulps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20205 %}
20206 ins_pipe( pipe_slow );
20207 %}
20208
20209 instruct vmulF_mem(vec dst, vec src, memory mem) %{
20210 predicate((UseAVX > 0) &&
20211 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20212 match(Set dst (MulVF src (LoadVector mem)));
20213 format %{ "vmulps $dst,$src,$mem\t! mul packedF" %}
20214 ins_encode %{
20215 int vlen_enc = vector_length_encoding(this);
20216 __ vmulps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20217 %}
20218 ins_pipe( pipe_slow );
20219 %}
20220
20221 // Doubles vector mul
20222 instruct vmulD(vec dst, vec src) %{
20223 predicate(UseAVX == 0);
20224 match(Set dst (MulVD dst src));
20225 format %{ "mulpd $dst,$src\t! mul packedD" %}
20226 ins_encode %{
20227 __ mulpd($dst$$XMMRegister, $src$$XMMRegister);
20228 %}
20229 ins_pipe( pipe_slow );
20230 %}
20231
20232 instruct vmulD_reg(vec dst, vec src1, vec src2) %{
20233 predicate(UseAVX > 0);
20234 match(Set dst (MulVD src1 src2));
20235 format %{ "vmulpd $dst,$src1,$src2\t! mul packedD" %}
20236 ins_encode %{
20237 int vlen_enc = vector_length_encoding(this);
20238 __ vmulpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20239 %}
20240 ins_pipe( pipe_slow );
20241 %}
20242
20243 instruct vmulD_mem(vec dst, vec src, memory mem) %{
20244 predicate((UseAVX > 0) &&
20245 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20246 match(Set dst (MulVD src (LoadVector mem)));
20247 format %{ "vmulpd $dst,$src,$mem\t! mul packedD" %}
20248 ins_encode %{
20249 int vlen_enc = vector_length_encoding(this);
20250 __ vmulpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20251 %}
20252 ins_pipe( pipe_slow );
20253 %}
20254
20255 // --------------------------------- DIV --------------------------------------
20256
20257 // Floats vector div
20258 instruct vdivF(vec dst, vec src) %{
20259 predicate(UseAVX == 0);
20260 match(Set dst (DivVF dst src));
20261 format %{ "divps $dst,$src\t! div packedF" %}
20262 ins_encode %{
20263 __ divps($dst$$XMMRegister, $src$$XMMRegister);
20264 %}
20265 ins_pipe( pipe_slow );
20266 %}
20267
20268 instruct vdivF_reg(vec dst, vec src1, vec src2) %{
20269 predicate(UseAVX > 0);
20270 match(Set dst (DivVF src1 src2));
20271 format %{ "vdivps $dst,$src1,$src2\t! div packedF" %}
20272 ins_encode %{
20273 int vlen_enc = vector_length_encoding(this);
20274 __ vdivps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20275 %}
20276 ins_pipe( pipe_slow );
20277 %}
20278
20279 instruct vdivF_mem(vec dst, vec src, memory mem) %{
20280 predicate((UseAVX > 0) &&
20281 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20282 match(Set dst (DivVF src (LoadVector mem)));
20283 format %{ "vdivps $dst,$src,$mem\t! div packedF" %}
20284 ins_encode %{
20285 int vlen_enc = vector_length_encoding(this);
20286 __ vdivps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20287 %}
20288 ins_pipe( pipe_slow );
20289 %}
20290
20291 // Doubles vector div
20292 instruct vdivD(vec dst, vec src) %{
20293 predicate(UseAVX == 0);
20294 match(Set dst (DivVD dst src));
20295 format %{ "divpd $dst,$src\t! div packedD" %}
20296 ins_encode %{
20297 __ divpd($dst$$XMMRegister, $src$$XMMRegister);
20298 %}
20299 ins_pipe( pipe_slow );
20300 %}
20301
20302 instruct vdivD_reg(vec dst, vec src1, vec src2) %{
20303 predicate(UseAVX > 0);
20304 match(Set dst (DivVD src1 src2));
20305 format %{ "vdivpd $dst,$src1,$src2\t! div packedD" %}
20306 ins_encode %{
20307 int vlen_enc = vector_length_encoding(this);
20308 __ vdivpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20309 %}
20310 ins_pipe( pipe_slow );
20311 %}
20312
20313 instruct vdivD_mem(vec dst, vec src, memory mem) %{
20314 predicate((UseAVX > 0) &&
20315 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20316 match(Set dst (DivVD src (LoadVector mem)));
20317 format %{ "vdivpd $dst,$src,$mem\t! div packedD" %}
20318 ins_encode %{
20319 int vlen_enc = vector_length_encoding(this);
20320 __ vdivpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20321 %}
20322 ins_pipe( pipe_slow );
20323 %}
20324
20325 // ------------------------------ MinMax ---------------------------------------
20326
20327 // Byte, Short, Int vector Min/Max
20328 instruct minmax_reg_sse(vec dst, vec src) %{
20329 predicate(is_integral_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_element_basic_type(n) != T_LONG && // T_BYTE, T_SHORT, T_INT
20330 UseAVX == 0);
20331 match(Set dst (MinV dst src));
20332 match(Set dst (MaxV dst src));
20333 format %{ "vector_minmax $dst,$src\t! " %}
20334 ins_encode %{
20335 assert(UseSSE >= 4, "required");
20336
20337 int opcode = this->ideal_Opcode();
20338 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20339 __ pminmax(opcode, elem_bt, $dst$$XMMRegister, $src$$XMMRegister);
20340 %}
20341 ins_pipe( pipe_slow );
20342 %}
20343
20344 instruct vminmax_reg(vec dst, vec src1, vec src2) %{
20345 predicate(is_integral_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_element_basic_type(n) != T_LONG && // T_BYTE, T_SHORT, T_INT
20346 UseAVX > 0);
20347 match(Set dst (MinV src1 src2));
20348 match(Set dst (MaxV src1 src2));
20349 format %{ "vector_minmax $dst,$src1,$src2\t! " %}
20350 ins_encode %{
20351 int opcode = this->ideal_Opcode();
20352 int vlen_enc = vector_length_encoding(this);
20353 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20354
20355 __ vpminmax(opcode, elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20356 %}
20357 ins_pipe( pipe_slow );
20358 %}
20359
20360 // Long vector Min/Max
20361 instruct minmaxL_reg_sse(vec dst, vec src, rxmm0 tmp) %{
20362 predicate(Matcher::vector_length_in_bytes(n) == 16 && Matcher::vector_element_basic_type(n) == T_LONG &&
20363 UseAVX == 0);
20364 match(Set dst (MinV dst src));
20365 match(Set dst (MaxV src dst));
20366 effect(TEMP dst, TEMP tmp);
20367 format %{ "vector_minmaxL $dst,$src\t!using $tmp as TEMP" %}
20368 ins_encode %{
20369 assert(UseSSE >= 4, "required");
20370
20371 int opcode = this->ideal_Opcode();
20372 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20373 assert(elem_bt == T_LONG, "sanity");
20374
20375 __ pminmax(opcode, elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $tmp$$XMMRegister);
20376 %}
20377 ins_pipe( pipe_slow );
20378 %}
20379
20380 instruct vminmaxL_reg_avx(legVec dst, legVec src1, legVec src2) %{
20381 predicate(Matcher::vector_length_in_bytes(n) <= 32 && Matcher::vector_element_basic_type(n) == T_LONG &&
20382 UseAVX > 0 && !VM_Version::supports_avx512vl());
20383 match(Set dst (MinV src1 src2));
20384 match(Set dst (MaxV src1 src2));
20385 effect(TEMP dst);
20386 format %{ "vector_minmaxL $dst,$src1,$src2\t! " %}
20387 ins_encode %{
20388 int vlen_enc = vector_length_encoding(this);
20389 int opcode = this->ideal_Opcode();
20390 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20391 assert(elem_bt == T_LONG, "sanity");
20392
20393 __ vpminmax(opcode, elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20394 %}
20395 ins_pipe( pipe_slow );
20396 %}
20397
20398 instruct vminmaxL_reg_evex(vec dst, vec src1, vec src2) %{
20399 predicate((Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()) &&
20400 Matcher::vector_element_basic_type(n) == T_LONG);
20401 match(Set dst (MinV src1 src2));
20402 match(Set dst (MaxV src1 src2));
20403 format %{ "vector_minmaxL $dst,$src1,src2\t! " %}
20404 ins_encode %{
20405 assert(UseAVX > 2, "required");
20406
20407 int vlen_enc = vector_length_encoding(this);
20408 int opcode = this->ideal_Opcode();
20409 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20410 assert(elem_bt == T_LONG, "sanity");
20411
20412 __ vpminmax(opcode, elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20413 %}
20414 ins_pipe( pipe_slow );
20415 %}
20416
20417 // Float/Double vector Min/Max
20418 instruct minmaxFP_reg_avx10_2(vec dst, vec a, vec b) %{
20419 predicate(VM_Version::supports_avx10_2() &&
20420 is_floating_point_type(Matcher::vector_element_basic_type(n))); // T_FLOAT, T_DOUBLE
20421 match(Set dst (MinV a b));
20422 match(Set dst (MaxV a b));
20423 format %{ "vector_minmaxFP $dst, $a, $b" %}
20424 ins_encode %{
20425 int vlen_enc = vector_length_encoding(this);
20426 int opcode = this->ideal_Opcode();
20427 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20428 __ vminmax_fp_avx10_2(opcode, elem_bt, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister, vlen_enc);
20429 %}
20430 ins_pipe( pipe_slow );
20431 %}
20432
20433 // Float/Double vector Min/Max
20434 instruct minmaxFP_reg(legVec dst, legVec a, legVec b, legVec tmp, legVec atmp, legVec btmp) %{
20435 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_length_in_bytes(n) <= 32 &&
20436 is_floating_point_type(Matcher::vector_element_basic_type(n)) && // T_FLOAT, T_DOUBLE
20437 UseAVX > 0);
20438 match(Set dst (MinV a b));
20439 match(Set dst (MaxV a b));
20440 effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
20441 format %{ "vector_minmaxFP $dst,$a,$b\t!using $tmp, $atmp, $btmp as TEMP" %}
20442 ins_encode %{
20443 assert(UseAVX > 0, "required");
20444
20445 int opcode = this->ideal_Opcode();
20446 int vlen_enc = vector_length_encoding(this);
20447 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20448
20449 __ vminmax_fp(opcode, elem_bt,
20450 $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister,
20451 $tmp$$XMMRegister, $atmp$$XMMRegister , $btmp$$XMMRegister, vlen_enc);
20452 %}
20453 ins_pipe( pipe_slow );
20454 %}
20455
20456 instruct evminmaxFP_reg_evex(vec dst, vec a, vec b, vec atmp, vec btmp, kReg ktmp) %{
20457 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_length_in_bytes(n) == 64 &&
20458 is_floating_point_type(Matcher::vector_element_basic_type(n))); // T_FLOAT, T_DOUBLE
20459 match(Set dst (MinV a b));
20460 match(Set dst (MaxV a b));
20461 effect(TEMP dst, USE a, USE b, TEMP atmp, TEMP btmp, TEMP ktmp);
20462 format %{ "vector_minmaxFP $dst,$a,$b\t!using $atmp, $btmp as TEMP" %}
20463 ins_encode %{
20464 assert(UseAVX > 2, "required");
20465
20466 int opcode = this->ideal_Opcode();
20467 int vlen_enc = vector_length_encoding(this);
20468 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20469
20470 __ evminmax_fp(opcode, elem_bt,
20471 $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister,
20472 $ktmp$$KRegister, $atmp$$XMMRegister , $btmp$$XMMRegister, vlen_enc);
20473 %}
20474 ins_pipe( pipe_slow );
20475 %}
20476
20477 // ------------------------------ Unsigned vector Min/Max ----------------------
20478
20479 instruct vector_uminmax_reg(vec dst, vec a, vec b) %{
20480 predicate(VM_Version::supports_avx512vl() || Matcher::vector_element_basic_type(n) != T_LONG);
20481 match(Set dst (UMinV a b));
20482 match(Set dst (UMaxV a b));
20483 format %{ "vector_uminmax $dst,$a,$b\t!" %}
20484 ins_encode %{
20485 int opcode = this->ideal_Opcode();
20486 int vlen_enc = vector_length_encoding(this);
20487 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20488 assert(is_integral_type(elem_bt), "");
20489 __ vpuminmax(opcode, elem_bt, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vlen_enc);
20490 %}
20491 ins_pipe( pipe_slow );
20492 %}
20493
20494 instruct vector_uminmax_mem(vec dst, vec a, memory b) %{
20495 predicate(VM_Version::supports_avx512vl() || Matcher::vector_element_basic_type(n) != T_LONG);
20496 match(Set dst (UMinV a (LoadVector b)));
20497 match(Set dst (UMaxV a (LoadVector b)));
20498 format %{ "vector_uminmax $dst,$a,$b\t!" %}
20499 ins_encode %{
20500 int opcode = this->ideal_Opcode();
20501 int vlen_enc = vector_length_encoding(this);
20502 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20503 assert(is_integral_type(elem_bt), "");
20504 __ vpuminmax(opcode, elem_bt, $dst$$XMMRegister, $a$$XMMRegister, $b$$Address, vlen_enc);
20505 %}
20506 ins_pipe( pipe_slow );
20507 %}
20508
20509 instruct vector_uminmaxq_reg(vec dst, vec a, vec b, vec xtmp1, vec xtmp2) %{
20510 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_element_basic_type(n) == T_LONG);
20511 match(Set dst (UMinV a b));
20512 match(Set dst (UMaxV a b));
20513 effect(TEMP xtmp1, TEMP xtmp2);
20514 format %{ "vector_uminmaxq $dst,$a,$b\t! using xtmp1 and xtmp2 as TEMP" %}
20515 ins_encode %{
20516 int opcode = this->ideal_Opcode();
20517 int vlen_enc = vector_length_encoding(this);
20518 __ vpuminmaxq(opcode, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20519 %}
20520 ins_pipe( pipe_slow );
20521 %}
20522
20523 instruct vector_uminmax_reg_masked(vec dst, vec src2, kReg mask) %{
20524 match(Set dst (UMinV (Binary dst src2) mask));
20525 match(Set dst (UMaxV (Binary dst src2) mask));
20526 format %{ "vector_uminmax_masked $dst, $dst, $src2, $mask\t! umin/max masked operation" %}
20527 ins_encode %{
20528 int vlen_enc = vector_length_encoding(this);
20529 BasicType bt = Matcher::vector_element_basic_type(this);
20530 int opc = this->ideal_Opcode();
20531 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
20532 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
20533 %}
20534 ins_pipe( pipe_slow );
20535 %}
20536
20537 instruct vector_uminmax_mem_masked(vec dst, memory src2, kReg mask) %{
20538 match(Set dst (UMinV (Binary dst (LoadVector src2)) mask));
20539 match(Set dst (UMaxV (Binary dst (LoadVector src2)) mask));
20540 format %{ "vector_uminmax_masked $dst, $dst, $src2, $mask\t! umin/max masked operation" %}
20541 ins_encode %{
20542 int vlen_enc = vector_length_encoding(this);
20543 BasicType bt = Matcher::vector_element_basic_type(this);
20544 int opc = this->ideal_Opcode();
20545 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
20546 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
20547 %}
20548 ins_pipe( pipe_slow );
20549 %}
20550
20551 // --------------------------------- Signum/CopySign ---------------------------
20552
20553 instruct signumF_reg(regF dst, regF zero, regF one, rFlagsReg cr) %{
20554 match(Set dst (SignumF dst (Binary zero one)));
20555 effect(KILL cr);
20556 format %{ "signumF $dst, $dst" %}
20557 ins_encode %{
20558 int opcode = this->ideal_Opcode();
20559 __ signum_fp(opcode, $dst$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister);
20560 %}
20561 ins_pipe( pipe_slow );
20562 %}
20563
20564 instruct signumD_reg(regD dst, regD zero, regD one, rFlagsReg cr) %{
20565 match(Set dst (SignumD dst (Binary zero one)));
20566 effect(KILL cr);
20567 format %{ "signumD $dst, $dst" %}
20568 ins_encode %{
20569 int opcode = this->ideal_Opcode();
20570 __ signum_fp(opcode, $dst$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister);
20571 %}
20572 ins_pipe( pipe_slow );
20573 %}
20574
20575 instruct signumV_reg_avx(vec dst, vec src, vec zero, vec one, vec xtmp1) %{
20576 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n) <= 32);
20577 match(Set dst (SignumVF src (Binary zero one)));
20578 match(Set dst (SignumVD src (Binary zero one)));
20579 effect(TEMP dst, TEMP xtmp1);
20580 format %{ "vector_signum_avx $dst, $src\t! using $xtmp1 as TEMP" %}
20581 ins_encode %{
20582 int opcode = this->ideal_Opcode();
20583 int vec_enc = vector_length_encoding(this);
20584 __ vector_signum_avx(opcode, $dst$$XMMRegister, $src$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister,
20585 $xtmp1$$XMMRegister, vec_enc);
20586 %}
20587 ins_pipe( pipe_slow );
20588 %}
20589
20590 instruct signumV_reg_evex(vec dst, vec src, vec zero, vec one, kReg ktmp1) %{
20591 predicate(VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64);
20592 match(Set dst (SignumVF src (Binary zero one)));
20593 match(Set dst (SignumVD src (Binary zero one)));
20594 effect(TEMP dst, TEMP ktmp1);
20595 format %{ "vector_signum_evex $dst, $src\t! using $ktmp1 as TEMP" %}
20596 ins_encode %{
20597 int opcode = this->ideal_Opcode();
20598 int vec_enc = vector_length_encoding(this);
20599 __ vector_signum_evex(opcode, $dst$$XMMRegister, $src$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister,
20600 $ktmp1$$KRegister, vec_enc);
20601 %}
20602 ins_pipe( pipe_slow );
20603 %}
20604
20605 // ---------------------------------------
20606 // For copySign use 0xE4 as writemask for vpternlog
20607 // Desired Truth Table: A -> xmm0 bit, B -> xmm1 bit, C -> xmm2 bit
20608 // C (xmm2) is set to 0x7FFFFFFF
20609 // Wherever xmm2 is 0, we want to pick from B (sign)
20610 // Wherever xmm2 is 1, we want to pick from A (src)
20611 //
20612 // A B C Result
20613 // 0 0 0 0
20614 // 0 0 1 0
20615 // 0 1 0 1
20616 // 0 1 1 0
20617 // 1 0 0 0
20618 // 1 0 1 1
20619 // 1 1 0 1
20620 // 1 1 1 1
20621 //
20622 // Result going from high bit to low bit is 0x11100100 = 0xe4
20623 // ---------------------------------------
20624
20625 instruct copySignF_reg(regF dst, regF src, regF tmp1, rRegI tmp2) %{
20626 match(Set dst (CopySignF dst src));
20627 effect(TEMP tmp1, TEMP tmp2);
20628 format %{ "CopySignF $dst, $src\t! using $tmp1 and $tmp2 as TEMP" %}
20629 ins_encode %{
20630 __ movl($tmp2$$Register, 0x7FFFFFFF);
20631 __ movdl($tmp1$$XMMRegister, $tmp2$$Register);
20632 __ vpternlogd($dst$$XMMRegister, 0xE4, $src$$XMMRegister, $tmp1$$XMMRegister, Assembler::AVX_128bit);
20633 %}
20634 ins_pipe( pipe_slow );
20635 %}
20636
20637 instruct copySignD_imm(regD dst, regD src, regD tmp1, rRegL tmp2, immD zero) %{
20638 match(Set dst (CopySignD dst (Binary src zero)));
20639 ins_cost(100);
20640 effect(TEMP tmp1, TEMP tmp2);
20641 format %{ "CopySignD $dst, $src\t! using $tmp1 and $tmp2 as TEMP" %}
20642 ins_encode %{
20643 __ mov64($tmp2$$Register, 0x7FFFFFFFFFFFFFFF);
20644 __ movq($tmp1$$XMMRegister, $tmp2$$Register);
20645 __ vpternlogq($dst$$XMMRegister, 0xE4, $src$$XMMRegister, $tmp1$$XMMRegister, Assembler::AVX_128bit);
20646 %}
20647 ins_pipe( pipe_slow );
20648 %}
20649
20650 //----------------------------- CompressBits/ExpandBits ------------------------
20651
20652 instruct compressBitsI_reg(rRegI dst, rRegI src, rRegI mask) %{
20653 predicate(n->bottom_type()->isa_int());
20654 match(Set dst (CompressBits src mask));
20655 format %{ "pextl $dst, $src, $mask\t! parallel bit extract" %}
20656 ins_encode %{
20657 __ pextl($dst$$Register, $src$$Register, $mask$$Register);
20658 %}
20659 ins_pipe( pipe_slow );
20660 %}
20661
20662 instruct expandBitsI_reg(rRegI dst, rRegI src, rRegI mask) %{
20663 predicate(n->bottom_type()->isa_int());
20664 match(Set dst (ExpandBits src mask));
20665 format %{ "pdepl $dst, $src, $mask\t! parallel bit deposit" %}
20666 ins_encode %{
20667 __ pdepl($dst$$Register, $src$$Register, $mask$$Register);
20668 %}
20669 ins_pipe( pipe_slow );
20670 %}
20671
20672 instruct compressBitsI_mem(rRegI dst, rRegI src, memory mask) %{
20673 predicate(n->bottom_type()->isa_int());
20674 match(Set dst (CompressBits src (LoadI mask)));
20675 format %{ "pextl $dst, $src, $mask\t! parallel bit extract" %}
20676 ins_encode %{
20677 __ pextl($dst$$Register, $src$$Register, $mask$$Address);
20678 %}
20679 ins_pipe( pipe_slow );
20680 %}
20681
20682 instruct expandBitsI_mem(rRegI dst, rRegI src, memory mask) %{
20683 predicate(n->bottom_type()->isa_int());
20684 match(Set dst (ExpandBits src (LoadI mask)));
20685 format %{ "pdepl $dst, $src, $mask\t! parallel bit deposit" %}
20686 ins_encode %{
20687 __ pdepl($dst$$Register, $src$$Register, $mask$$Address);
20688 %}
20689 ins_pipe( pipe_slow );
20690 %}
20691
20692 // --------------------------------- Sqrt --------------------------------------
20693
20694 instruct vsqrtF_reg(vec dst, vec src) %{
20695 match(Set dst (SqrtVF src));
20696 format %{ "vsqrtps $dst,$src\t! sqrt packedF" %}
20697 ins_encode %{
20698 assert(UseAVX > 0, "required");
20699 int vlen_enc = vector_length_encoding(this);
20700 __ vsqrtps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
20701 %}
20702 ins_pipe( pipe_slow );
20703 %}
20704
20705 instruct vsqrtF_mem(vec dst, memory mem) %{
20706 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
20707 match(Set dst (SqrtVF (LoadVector mem)));
20708 format %{ "vsqrtps $dst,$mem\t! sqrt packedF" %}
20709 ins_encode %{
20710 assert(UseAVX > 0, "required");
20711 int vlen_enc = vector_length_encoding(this);
20712 __ vsqrtps($dst$$XMMRegister, $mem$$Address, vlen_enc);
20713 %}
20714 ins_pipe( pipe_slow );
20715 %}
20716
20717 // Floating point vector sqrt
20718 instruct vsqrtD_reg(vec dst, vec src) %{
20719 match(Set dst (SqrtVD src));
20720 format %{ "vsqrtpd $dst,$src\t! sqrt packedD" %}
20721 ins_encode %{
20722 assert(UseAVX > 0, "required");
20723 int vlen_enc = vector_length_encoding(this);
20724 __ vsqrtpd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
20725 %}
20726 ins_pipe( pipe_slow );
20727 %}
20728
20729 instruct vsqrtD_mem(vec dst, memory mem) %{
20730 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
20731 match(Set dst (SqrtVD (LoadVector mem)));
20732 format %{ "vsqrtpd $dst,$mem\t! sqrt packedD" %}
20733 ins_encode %{
20734 assert(UseAVX > 0, "required");
20735 int vlen_enc = vector_length_encoding(this);
20736 __ vsqrtpd($dst$$XMMRegister, $mem$$Address, vlen_enc);
20737 %}
20738 ins_pipe( pipe_slow );
20739 %}
20740
20741 // ------------------------------ Shift ---------------------------------------
20742
20743 // Left and right shift count vectors are the same on x86
20744 // (only lowest bits of xmm reg are used for count).
20745 instruct vshiftcnt(vec dst, rRegI cnt) %{
20746 match(Set dst (LShiftCntV cnt));
20747 match(Set dst (RShiftCntV cnt));
20748 format %{ "movdl $dst,$cnt\t! load shift count" %}
20749 ins_encode %{
20750 __ movdl($dst$$XMMRegister, $cnt$$Register);
20751 %}
20752 ins_pipe( pipe_slow );
20753 %}
20754
20755 // Byte vector shift
20756 instruct vshiftB(vec dst, vec src, vec shift, vec tmp) %{
20757 predicate(Matcher::vector_length(n) <= 8 && !n->as_ShiftV()->is_var_shift());
20758 match(Set dst ( LShiftVB src shift));
20759 match(Set dst ( RShiftVB src shift));
20760 match(Set dst (URShiftVB src shift));
20761 effect(TEMP dst, USE src, USE shift, TEMP tmp);
20762 format %{"vector_byte_shift $dst,$src,$shift" %}
20763 ins_encode %{
20764 assert(UseSSE > 3, "required");
20765 int opcode = this->ideal_Opcode();
20766 bool sign = (opcode != Op_URShiftVB);
20767 __ vextendbw(sign, $tmp$$XMMRegister, $src$$XMMRegister);
20768 __ vshiftw(opcode, $tmp$$XMMRegister, $shift$$XMMRegister);
20769 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), noreg);
20770 __ pand($dst$$XMMRegister, $tmp$$XMMRegister);
20771 __ packuswb($dst$$XMMRegister, $dst$$XMMRegister);
20772 %}
20773 ins_pipe( pipe_slow );
20774 %}
20775
20776 instruct vshift16B(vec dst, vec src, vec shift, vec tmp1, vec tmp2) %{
20777 predicate(Matcher::vector_length(n) == 16 && !n->as_ShiftV()->is_var_shift() &&
20778 UseAVX <= 1);
20779 match(Set dst ( LShiftVB src shift));
20780 match(Set dst ( RShiftVB src shift));
20781 match(Set dst (URShiftVB src shift));
20782 effect(TEMP dst, USE src, USE shift, TEMP tmp1, TEMP tmp2);
20783 format %{"vector_byte_shift $dst,$src,$shift" %}
20784 ins_encode %{
20785 assert(UseSSE > 3, "required");
20786 int opcode = this->ideal_Opcode();
20787 bool sign = (opcode != Op_URShiftVB);
20788 __ vextendbw(sign, $tmp1$$XMMRegister, $src$$XMMRegister);
20789 __ vshiftw(opcode, $tmp1$$XMMRegister, $shift$$XMMRegister);
20790 __ pshufd($tmp2$$XMMRegister, $src$$XMMRegister, 0xE);
20791 __ vextendbw(sign, $tmp2$$XMMRegister, $tmp2$$XMMRegister);
20792 __ vshiftw(opcode, $tmp2$$XMMRegister, $shift$$XMMRegister);
20793 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), noreg);
20794 __ pand($tmp2$$XMMRegister, $dst$$XMMRegister);
20795 __ pand($dst$$XMMRegister, $tmp1$$XMMRegister);
20796 __ packuswb($dst$$XMMRegister, $tmp2$$XMMRegister);
20797 %}
20798 ins_pipe( pipe_slow );
20799 %}
20800
20801 instruct vshift16B_avx(vec dst, vec src, vec shift, vec tmp) %{
20802 predicate(Matcher::vector_length(n) == 16 && !n->as_ShiftV()->is_var_shift() &&
20803 UseAVX > 1);
20804 match(Set dst ( LShiftVB src shift));
20805 match(Set dst ( RShiftVB src shift));
20806 match(Set dst (URShiftVB src shift));
20807 effect(TEMP dst, TEMP tmp);
20808 format %{"vector_byte_shift $dst,$src,$shift" %}
20809 ins_encode %{
20810 int opcode = this->ideal_Opcode();
20811 bool sign = (opcode != Op_URShiftVB);
20812 int vlen_enc = Assembler::AVX_256bit;
20813 __ vextendbw(sign, $tmp$$XMMRegister, $src$$XMMRegister, vlen_enc);
20814 __ vshiftw(opcode, $tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20815 __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
20816 __ vextracti128_high($dst$$XMMRegister, $tmp$$XMMRegister);
20817 __ vpackuswb($dst$$XMMRegister, $tmp$$XMMRegister, $dst$$XMMRegister, 0);
20818 %}
20819 ins_pipe( pipe_slow );
20820 %}
20821
20822 instruct vshift32B_avx(vec dst, vec src, vec shift, vec tmp) %{
20823 predicate(Matcher::vector_length(n) == 32 && !n->as_ShiftV()->is_var_shift());
20824 match(Set dst ( LShiftVB src shift));
20825 match(Set dst ( RShiftVB src shift));
20826 match(Set dst (URShiftVB src shift));
20827 effect(TEMP dst, TEMP tmp);
20828 format %{"vector_byte_shift $dst,$src,$shift" %}
20829 ins_encode %{
20830 assert(UseAVX > 1, "required");
20831 int opcode = this->ideal_Opcode();
20832 bool sign = (opcode != Op_URShiftVB);
20833 int vlen_enc = Assembler::AVX_256bit;
20834 __ vextracti128_high($tmp$$XMMRegister, $src$$XMMRegister);
20835 __ vextendbw(sign, $tmp$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
20836 __ vextendbw(sign, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
20837 __ vshiftw(opcode, $tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20838 __ vshiftw(opcode, $dst$$XMMRegister, $dst$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20839 __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
20840 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
20841 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
20842 __ vpermq($dst$$XMMRegister, $dst$$XMMRegister, 0xD8, vlen_enc);
20843 %}
20844 ins_pipe( pipe_slow );
20845 %}
20846
20847 instruct vshift64B_avx(vec dst, vec src, vec shift, vec tmp1, vec tmp2) %{
20848 predicate(Matcher::vector_length(n) == 64 && !n->as_ShiftV()->is_var_shift());
20849 match(Set dst ( LShiftVB src shift));
20850 match(Set dst (RShiftVB src shift));
20851 match(Set dst (URShiftVB src shift));
20852 effect(TEMP dst, TEMP tmp1, TEMP tmp2);
20853 format %{"vector_byte_shift $dst,$src,$shift" %}
20854 ins_encode %{
20855 assert(UseAVX > 2, "required");
20856 int opcode = this->ideal_Opcode();
20857 bool sign = (opcode != Op_URShiftVB);
20858 int vlen_enc = Assembler::AVX_512bit;
20859 __ vextracti64x4($tmp1$$XMMRegister, $src$$XMMRegister, 1);
20860 __ vextendbw(sign, $tmp1$$XMMRegister, $tmp1$$XMMRegister, vlen_enc);
20861 __ vextendbw(sign, $tmp2$$XMMRegister, $src$$XMMRegister, vlen_enc);
20862 __ vshiftw(opcode, $tmp1$$XMMRegister, $tmp1$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20863 __ vshiftw(opcode, $tmp2$$XMMRegister, $tmp2$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20864 __ vmovdqu($dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), noreg);
20865 __ vpbroadcastd($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
20866 __ vpand($tmp1$$XMMRegister, $tmp1$$XMMRegister, $dst$$XMMRegister, vlen_enc);
20867 __ vpand($tmp2$$XMMRegister, $tmp2$$XMMRegister, $dst$$XMMRegister, vlen_enc);
20868 __ vpackuswb($dst$$XMMRegister, $tmp1$$XMMRegister, $tmp2$$XMMRegister, vlen_enc);
20869 __ evmovdquq($tmp2$$XMMRegister, ExternalAddress(vector_byte_perm_mask()), vlen_enc, noreg);
20870 __ vpermq($dst$$XMMRegister, $tmp2$$XMMRegister, $dst$$XMMRegister, vlen_enc);
20871 %}
20872 ins_pipe( pipe_slow );
20873 %}
20874
20875 // Shorts vector logical right shift produces incorrect Java result
20876 // for negative data because java code convert short value into int with
20877 // sign extension before a shift. But char vectors are fine since chars are
20878 // unsigned values.
20879 // Shorts/Chars vector left shift
20880 instruct vshiftS(vec dst, vec src, vec shift) %{
20881 predicate(!n->as_ShiftV()->is_var_shift());
20882 match(Set dst ( LShiftVS src shift));
20883 match(Set dst ( RShiftVS src shift));
20884 match(Set dst (URShiftVS src shift));
20885 effect(TEMP dst, USE src, USE shift);
20886 format %{ "vshiftw $dst,$src,$shift\t! shift packedS" %}
20887 ins_encode %{
20888 int opcode = this->ideal_Opcode();
20889 if (UseAVX > 0) {
20890 int vlen_enc = vector_length_encoding(this);
20891 __ vshiftw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20892 } else {
20893 int vlen = Matcher::vector_length(this);
20894 if (vlen == 2) {
20895 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
20896 __ vshiftw(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
20897 } else if (vlen == 4) {
20898 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
20899 __ vshiftw(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
20900 } else {
20901 assert (vlen == 8, "sanity");
20902 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
20903 __ vshiftw(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
20904 }
20905 }
20906 %}
20907 ins_pipe( pipe_slow );
20908 %}
20909
20910 // Integers vector left shift
20911 instruct vshiftI(vec dst, vec src, vec shift) %{
20912 predicate(!n->as_ShiftV()->is_var_shift());
20913 match(Set dst ( LShiftVI src shift));
20914 match(Set dst ( RShiftVI src shift));
20915 match(Set dst (URShiftVI src shift));
20916 effect(TEMP dst, USE src, USE shift);
20917 format %{ "vshiftd $dst,$src,$shift\t! shift packedI" %}
20918 ins_encode %{
20919 int opcode = this->ideal_Opcode();
20920 if (UseAVX > 0) {
20921 int vlen_enc = vector_length_encoding(this);
20922 __ vshiftd(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20923 } else {
20924 int vlen = Matcher::vector_length(this);
20925 if (vlen == 2) {
20926 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
20927 __ vshiftd(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
20928 } else {
20929 assert(vlen == 4, "sanity");
20930 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
20931 __ vshiftd(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
20932 }
20933 }
20934 %}
20935 ins_pipe( pipe_slow );
20936 %}
20937
20938 // Integers vector left constant shift
20939 instruct vshiftI_imm(vec dst, vec src, immI8 shift) %{
20940 match(Set dst (LShiftVI src (LShiftCntV shift)));
20941 match(Set dst (RShiftVI src (RShiftCntV shift)));
20942 match(Set dst (URShiftVI src (RShiftCntV shift)));
20943 format %{ "vshiftd_imm $dst,$src,$shift\t! shift packedI" %}
20944 ins_encode %{
20945 int opcode = this->ideal_Opcode();
20946 if (UseAVX > 0) {
20947 int vector_len = vector_length_encoding(this);
20948 __ vshiftd_imm(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$constant, vector_len);
20949 } else {
20950 int vlen = Matcher::vector_length(this);
20951 if (vlen == 2) {
20952 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
20953 __ vshiftd_imm(opcode, $dst$$XMMRegister, $shift$$constant);
20954 } else {
20955 assert(vlen == 4, "sanity");
20956 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
20957 __ vshiftd_imm(opcode, $dst$$XMMRegister, $shift$$constant);
20958 }
20959 }
20960 %}
20961 ins_pipe( pipe_slow );
20962 %}
20963
20964 // Longs vector shift
20965 instruct vshiftL(vec dst, vec src, vec shift) %{
20966 predicate(!n->as_ShiftV()->is_var_shift());
20967 match(Set dst ( LShiftVL src shift));
20968 match(Set dst (URShiftVL src shift));
20969 effect(TEMP dst, USE src, USE shift);
20970 format %{ "vshiftq $dst,$src,$shift\t! shift packedL" %}
20971 ins_encode %{
20972 int opcode = this->ideal_Opcode();
20973 if (UseAVX > 0) {
20974 int vlen_enc = vector_length_encoding(this);
20975 __ vshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20976 } else {
20977 assert(Matcher::vector_length(this) == 2, "");
20978 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
20979 __ vshiftq(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
20980 }
20981 %}
20982 ins_pipe( pipe_slow );
20983 %}
20984
20985 // Longs vector constant shift
20986 instruct vshiftL_imm(vec dst, vec src, immI8 shift) %{
20987 match(Set dst (LShiftVL src (LShiftCntV shift)));
20988 match(Set dst (URShiftVL src (RShiftCntV shift)));
20989 format %{ "vshiftq_imm $dst,$src,$shift\t! shift packedL" %}
20990 ins_encode %{
20991 int opcode = this->ideal_Opcode();
20992 if (UseAVX > 0) {
20993 int vector_len = vector_length_encoding(this);
20994 __ vshiftq_imm(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$constant, vector_len);
20995 } else {
20996 assert(Matcher::vector_length(this) == 2, "");
20997 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
20998 __ vshiftq_imm(opcode, $dst$$XMMRegister, $shift$$constant);
20999 }
21000 %}
21001 ins_pipe( pipe_slow );
21002 %}
21003
21004 // -------------------ArithmeticRightShift -----------------------------------
21005 // Long vector arithmetic right shift
21006 instruct vshiftL_arith_reg(vec dst, vec src, vec shift, vec tmp) %{
21007 predicate(!n->as_ShiftV()->is_var_shift() && UseAVX <= 2);
21008 match(Set dst (RShiftVL src shift));
21009 effect(TEMP dst, TEMP tmp);
21010 format %{ "vshiftq $dst,$src,$shift" %}
21011 ins_encode %{
21012 uint vlen = Matcher::vector_length(this);
21013 if (vlen == 2) {
21014 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
21015 __ psrlq($dst$$XMMRegister, $shift$$XMMRegister);
21016 __ movdqu($tmp$$XMMRegister, ExternalAddress(vector_long_sign_mask()), noreg);
21017 __ psrlq($tmp$$XMMRegister, $shift$$XMMRegister);
21018 __ pxor($dst$$XMMRegister, $tmp$$XMMRegister);
21019 __ psubq($dst$$XMMRegister, $tmp$$XMMRegister);
21020 } else {
21021 assert(vlen == 4, "sanity");
21022 assert(UseAVX > 1, "required");
21023 int vlen_enc = Assembler::AVX_256bit;
21024 __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21025 __ vmovdqu($tmp$$XMMRegister, ExternalAddress(vector_long_sign_mask()), noreg);
21026 __ vpsrlq($tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21027 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
21028 __ vpsubq($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
21029 }
21030 %}
21031 ins_pipe( pipe_slow );
21032 %}
21033
21034 instruct vshiftL_arith_reg_evex(vec dst, vec src, vec shift) %{
21035 predicate(!n->as_ShiftV()->is_var_shift() && UseAVX > 2);
21036 match(Set dst (RShiftVL src shift));
21037 format %{ "vshiftq $dst,$src,$shift" %}
21038 ins_encode %{
21039 int vlen_enc = vector_length_encoding(this);
21040 __ evpsraq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21041 %}
21042 ins_pipe( pipe_slow );
21043 %}
21044
21045 // ------------------- Variable Shift -----------------------------
21046 // Byte variable shift
21047 instruct vshift8B_var_nobw(vec dst, vec src, vec shift, vec vtmp) %{
21048 predicate(Matcher::vector_length(n) <= 8 &&
21049 n->as_ShiftV()->is_var_shift() &&
21050 !VM_Version::supports_avx512bw());
21051 match(Set dst ( LShiftVB src shift));
21052 match(Set dst ( RShiftVB src shift));
21053 match(Set dst (URShiftVB src shift));
21054 effect(TEMP dst, TEMP vtmp);
21055 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp as TEMP" %}
21056 ins_encode %{
21057 assert(UseAVX >= 2, "required");
21058
21059 int opcode = this->ideal_Opcode();
21060 int vlen_enc = Assembler::AVX_128bit;
21061 __ varshiftbw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp$$XMMRegister);
21062 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, 0);
21063 %}
21064 ins_pipe( pipe_slow );
21065 %}
21066
21067 instruct vshift16B_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2) %{
21068 predicate(Matcher::vector_length(n) == 16 &&
21069 n->as_ShiftV()->is_var_shift() &&
21070 !VM_Version::supports_avx512bw());
21071 match(Set dst ( LShiftVB src shift));
21072 match(Set dst ( RShiftVB src shift));
21073 match(Set dst (URShiftVB src shift));
21074 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
21075 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp1, $vtmp2 as TEMP" %}
21076 ins_encode %{
21077 assert(UseAVX >= 2, "required");
21078
21079 int opcode = this->ideal_Opcode();
21080 int vlen_enc = Assembler::AVX_128bit;
21081 // Shift lower half and get word result in dst
21082 __ varshiftbw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp1$$XMMRegister);
21083
21084 // Shift upper half and get word result in vtmp1
21085 __ vpshufd($vtmp1$$XMMRegister, $src$$XMMRegister, 0xE, 0);
21086 __ vpshufd($vtmp2$$XMMRegister, $shift$$XMMRegister, 0xE, 0);
21087 __ varshiftbw(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21088
21089 // Merge and down convert the two word results to byte in dst
21090 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0);
21091 %}
21092 ins_pipe( pipe_slow );
21093 %}
21094
21095 instruct vshift32B_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2, vec vtmp3, vec vtmp4) %{
21096 predicate(Matcher::vector_length(n) == 32 &&
21097 n->as_ShiftV()->is_var_shift() &&
21098 !VM_Version::supports_avx512bw());
21099 match(Set dst ( LShiftVB src shift));
21100 match(Set dst ( RShiftVB src shift));
21101 match(Set dst (URShiftVB src shift));
21102 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4);
21103 format %{ "vector_varshift_byte $dst, $src, $shift\n\t using $vtmp1, $vtmp2, $vtmp3, $vtmp4 as TEMP" %}
21104 ins_encode %{
21105 assert(UseAVX >= 2, "required");
21106
21107 int opcode = this->ideal_Opcode();
21108 int vlen_enc = Assembler::AVX_128bit;
21109 // Process lower 128 bits and get result in dst
21110 __ varshiftbw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp1$$XMMRegister);
21111 __ vpshufd($vtmp1$$XMMRegister, $src$$XMMRegister, 0xE, 0);
21112 __ vpshufd($vtmp2$$XMMRegister, $shift$$XMMRegister, 0xE, 0);
21113 __ varshiftbw(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21114 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0);
21115
21116 // Process higher 128 bits and get result in vtmp3
21117 __ vextracti128_high($vtmp1$$XMMRegister, $src$$XMMRegister);
21118 __ vextracti128_high($vtmp2$$XMMRegister, $shift$$XMMRegister);
21119 __ varshiftbw(opcode, $vtmp3$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp4$$XMMRegister);
21120 __ vpshufd($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, 0xE, 0);
21121 __ vpshufd($vtmp2$$XMMRegister, $vtmp2$$XMMRegister, 0xE, 0);
21122 __ varshiftbw(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21123 __ vpackuswb($vtmp1$$XMMRegister, $vtmp3$$XMMRegister, $vtmp1$$XMMRegister, 0);
21124
21125 // Merge the two results in dst
21126 __ vinserti128($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0x1);
21127 %}
21128 ins_pipe( pipe_slow );
21129 %}
21130
21131 instruct vshiftB_var_evex_bw(vec dst, vec src, vec shift, vec vtmp) %{
21132 predicate(Matcher::vector_length(n) <= 32 &&
21133 n->as_ShiftV()->is_var_shift() &&
21134 VM_Version::supports_avx512bw());
21135 match(Set dst ( LShiftVB src shift));
21136 match(Set dst ( RShiftVB src shift));
21137 match(Set dst (URShiftVB src shift));
21138 effect(TEMP dst, TEMP vtmp);
21139 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp as TEMP" %}
21140 ins_encode %{
21141 assert(UseAVX > 2, "required");
21142
21143 int opcode = this->ideal_Opcode();
21144 int vlen_enc = vector_length_encoding(this);
21145 __ evarshiftb(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp$$XMMRegister);
21146 %}
21147 ins_pipe( pipe_slow );
21148 %}
21149
21150 instruct vshift64B_var_evex_bw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2) %{
21151 predicate(Matcher::vector_length(n) == 64 &&
21152 n->as_ShiftV()->is_var_shift() &&
21153 VM_Version::supports_avx512bw());
21154 match(Set dst ( LShiftVB src shift));
21155 match(Set dst ( RShiftVB src shift));
21156 match(Set dst (URShiftVB src shift));
21157 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
21158 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp1, $vtmp2 as TEMP" %}
21159 ins_encode %{
21160 assert(UseAVX > 2, "required");
21161
21162 int opcode = this->ideal_Opcode();
21163 int vlen_enc = Assembler::AVX_256bit;
21164 __ evarshiftb(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp1$$XMMRegister);
21165 __ vextracti64x4_high($vtmp1$$XMMRegister, $src$$XMMRegister);
21166 __ vextracti64x4_high($vtmp2$$XMMRegister, $shift$$XMMRegister);
21167 __ evarshiftb(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21168 __ vinserti64x4($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0x1);
21169 %}
21170 ins_pipe( pipe_slow );
21171 %}
21172
21173 // Short variable shift
21174 instruct vshift8S_var_nobw(vec dst, vec src, vec shift, vec vtmp) %{
21175 predicate(Matcher::vector_length(n) <= 8 &&
21176 n->as_ShiftV()->is_var_shift() &&
21177 !VM_Version::supports_avx512bw());
21178 match(Set dst ( LShiftVS src shift));
21179 match(Set dst ( RShiftVS src shift));
21180 match(Set dst (URShiftVS src shift));
21181 effect(TEMP dst, TEMP vtmp);
21182 format %{ "vector_var_shift_left_short $dst, $src, $shift\n\t" %}
21183 ins_encode %{
21184 assert(UseAVX >= 2, "required");
21185
21186 int opcode = this->ideal_Opcode();
21187 bool sign = (opcode != Op_URShiftVS);
21188 int vlen_enc = Assembler::AVX_256bit;
21189 __ vextendwd(sign, $dst$$XMMRegister, $src$$XMMRegister, 1);
21190 __ vpmovzxwd($vtmp$$XMMRegister, $shift$$XMMRegister, 1);
21191 __ varshiftd(opcode, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
21192 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21193 __ vextracti128_high($vtmp$$XMMRegister, $dst$$XMMRegister);
21194 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0);
21195 %}
21196 ins_pipe( pipe_slow );
21197 %}
21198
21199 instruct vshift16S_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2) %{
21200 predicate(Matcher::vector_length(n) == 16 &&
21201 n->as_ShiftV()->is_var_shift() &&
21202 !VM_Version::supports_avx512bw());
21203 match(Set dst ( LShiftVS src shift));
21204 match(Set dst ( RShiftVS src shift));
21205 match(Set dst (URShiftVS src shift));
21206 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
21207 format %{ "vector_var_shift_left_short $dst, $src, $shift\n\t" %}
21208 ins_encode %{
21209 assert(UseAVX >= 2, "required");
21210
21211 int opcode = this->ideal_Opcode();
21212 bool sign = (opcode != Op_URShiftVS);
21213 int vlen_enc = Assembler::AVX_256bit;
21214 // Shift lower half, with result in vtmp2 using vtmp1 as TEMP
21215 __ vextendwd(sign, $vtmp2$$XMMRegister, $src$$XMMRegister, vlen_enc);
21216 __ vpmovzxwd($vtmp1$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21217 __ varshiftd(opcode, $vtmp2$$XMMRegister, $vtmp2$$XMMRegister, $vtmp1$$XMMRegister, vlen_enc);
21218 __ vpand($vtmp2$$XMMRegister, $vtmp2$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21219
21220 // Shift upper half, with result in dst using vtmp1 as TEMP
21221 __ vextracti128_high($dst$$XMMRegister, $src$$XMMRegister);
21222 __ vextracti128_high($vtmp1$$XMMRegister, $shift$$XMMRegister);
21223 __ vextendwd(sign, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21224 __ vpmovzxwd($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, vlen_enc);
21225 __ varshiftd(opcode, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, vlen_enc);
21226 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21227
21228 // Merge lower and upper half result into dst
21229 __ vpackusdw($dst$$XMMRegister, $vtmp2$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21230 __ vpermq($dst$$XMMRegister, $dst$$XMMRegister, 0xD8, vlen_enc);
21231 %}
21232 ins_pipe( pipe_slow );
21233 %}
21234
21235 instruct vshift16S_var_evex_bw(vec dst, vec src, vec shift) %{
21236 predicate(n->as_ShiftV()->is_var_shift() &&
21237 VM_Version::supports_avx512bw());
21238 match(Set dst ( LShiftVS src shift));
21239 match(Set dst ( RShiftVS src shift));
21240 match(Set dst (URShiftVS src shift));
21241 format %{ "vector_varshift_short $dst,$src,$shift\t!" %}
21242 ins_encode %{
21243 assert(UseAVX > 2, "required");
21244
21245 int opcode = this->ideal_Opcode();
21246 int vlen_enc = vector_length_encoding(this);
21247 if (!VM_Version::supports_avx512vl()) {
21248 vlen_enc = Assembler::AVX_512bit;
21249 }
21250 __ varshiftw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21251 %}
21252 ins_pipe( pipe_slow );
21253 %}
21254
21255 //Integer variable shift
21256 instruct vshiftI_var(vec dst, vec src, vec shift) %{
21257 predicate(n->as_ShiftV()->is_var_shift());
21258 match(Set dst ( LShiftVI src shift));
21259 match(Set dst ( RShiftVI src shift));
21260 match(Set dst (URShiftVI src shift));
21261 format %{ "vector_varshift_int $dst,$src,$shift\t!" %}
21262 ins_encode %{
21263 assert(UseAVX >= 2, "required");
21264
21265 int opcode = this->ideal_Opcode();
21266 int vlen_enc = vector_length_encoding(this);
21267 __ varshiftd(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21268 %}
21269 ins_pipe( pipe_slow );
21270 %}
21271
21272 //Long variable shift
21273 instruct vshiftL_var(vec dst, vec src, vec shift) %{
21274 predicate(n->as_ShiftV()->is_var_shift());
21275 match(Set dst ( LShiftVL src shift));
21276 match(Set dst (URShiftVL src shift));
21277 format %{ "vector_varshift_long $dst,$src,$shift\t!" %}
21278 ins_encode %{
21279 assert(UseAVX >= 2, "required");
21280
21281 int opcode = this->ideal_Opcode();
21282 int vlen_enc = vector_length_encoding(this);
21283 __ varshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21284 %}
21285 ins_pipe( pipe_slow );
21286 %}
21287
21288 //Long variable right shift arithmetic
21289 instruct vshiftL_arith_var(vec dst, vec src, vec shift, vec vtmp) %{
21290 predicate(Matcher::vector_length(n) <= 4 &&
21291 n->as_ShiftV()->is_var_shift() &&
21292 UseAVX == 2);
21293 match(Set dst (RShiftVL src shift));
21294 effect(TEMP dst, TEMP vtmp);
21295 format %{ "vector_varshift_long $dst,$src,$shift\n\t! using $vtmp as TEMP" %}
21296 ins_encode %{
21297 int opcode = this->ideal_Opcode();
21298 int vlen_enc = vector_length_encoding(this);
21299 __ varshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc,
21300 $vtmp$$XMMRegister);
21301 %}
21302 ins_pipe( pipe_slow );
21303 %}
21304
21305 instruct vshiftL_arith_var_evex(vec dst, vec src, vec shift) %{
21306 predicate(n->as_ShiftV()->is_var_shift() &&
21307 UseAVX > 2);
21308 match(Set dst (RShiftVL src shift));
21309 format %{ "vector_varfshift_long $dst,$src,$shift\t!" %}
21310 ins_encode %{
21311 int opcode = this->ideal_Opcode();
21312 int vlen_enc = vector_length_encoding(this);
21313 __ varshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21314 %}
21315 ins_pipe( pipe_slow );
21316 %}
21317
21318 // --------------------------------- AND --------------------------------------
21319
21320 instruct vand(vec dst, vec src) %{
21321 predicate(UseAVX == 0);
21322 match(Set dst (AndV dst src));
21323 format %{ "pand $dst,$src\t! and vectors" %}
21324 ins_encode %{
21325 __ pand($dst$$XMMRegister, $src$$XMMRegister);
21326 %}
21327 ins_pipe( pipe_slow );
21328 %}
21329
21330 instruct vand_reg(vec dst, vec src1, vec src2) %{
21331 predicate(UseAVX > 0);
21332 match(Set dst (AndV src1 src2));
21333 format %{ "vpand $dst,$src1,$src2\t! and vectors" %}
21334 ins_encode %{
21335 int vlen_enc = vector_length_encoding(this);
21336 __ vpand($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
21337 %}
21338 ins_pipe( pipe_slow );
21339 %}
21340
21341 instruct vand_mem(vec dst, vec src, memory mem) %{
21342 predicate((UseAVX > 0) &&
21343 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
21344 match(Set dst (AndV src (LoadVector mem)));
21345 format %{ "vpand $dst,$src,$mem\t! and vectors" %}
21346 ins_encode %{
21347 int vlen_enc = vector_length_encoding(this);
21348 __ vpand($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
21349 %}
21350 ins_pipe( pipe_slow );
21351 %}
21352
21353 // --------------------------------- OR ---------------------------------------
21354
21355 instruct vor(vec dst, vec src) %{
21356 predicate(UseAVX == 0);
21357 match(Set dst (OrV dst src));
21358 format %{ "por $dst,$src\t! or vectors" %}
21359 ins_encode %{
21360 __ por($dst$$XMMRegister, $src$$XMMRegister);
21361 %}
21362 ins_pipe( pipe_slow );
21363 %}
21364
21365 instruct vor_reg(vec dst, vec src1, vec src2) %{
21366 predicate(UseAVX > 0);
21367 match(Set dst (OrV src1 src2));
21368 format %{ "vpor $dst,$src1,$src2\t! or vectors" %}
21369 ins_encode %{
21370 int vlen_enc = vector_length_encoding(this);
21371 __ vpor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
21372 %}
21373 ins_pipe( pipe_slow );
21374 %}
21375
21376 instruct vor_mem(vec dst, vec src, memory mem) %{
21377 predicate((UseAVX > 0) &&
21378 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
21379 match(Set dst (OrV src (LoadVector mem)));
21380 format %{ "vpor $dst,$src,$mem\t! or vectors" %}
21381 ins_encode %{
21382 int vlen_enc = vector_length_encoding(this);
21383 __ vpor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
21384 %}
21385 ins_pipe( pipe_slow );
21386 %}
21387
21388 // --------------------------------- XOR --------------------------------------
21389
21390 instruct vxor(vec dst, vec src) %{
21391 predicate(UseAVX == 0);
21392 match(Set dst (XorV dst src));
21393 format %{ "pxor $dst,$src\t! xor vectors" %}
21394 ins_encode %{
21395 __ pxor($dst$$XMMRegister, $src$$XMMRegister);
21396 %}
21397 ins_pipe( pipe_slow );
21398 %}
21399
21400 instruct vxor_reg(vec dst, vec src1, vec src2) %{
21401 predicate(UseAVX > 0);
21402 match(Set dst (XorV src1 src2));
21403 format %{ "vpxor $dst,$src1,$src2\t! xor vectors" %}
21404 ins_encode %{
21405 int vlen_enc = vector_length_encoding(this);
21406 __ vpxor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
21407 %}
21408 ins_pipe( pipe_slow );
21409 %}
21410
21411 instruct vxor_mem(vec dst, vec src, memory mem) %{
21412 predicate((UseAVX > 0) &&
21413 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
21414 match(Set dst (XorV src (LoadVector mem)));
21415 format %{ "vpxor $dst,$src,$mem\t! xor vectors" %}
21416 ins_encode %{
21417 int vlen_enc = vector_length_encoding(this);
21418 __ vpxor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
21419 %}
21420 ins_pipe( pipe_slow );
21421 %}
21422
21423 // --------------------------------- VectorCast --------------------------------------
21424
21425 instruct vcastBtoX(vec dst, vec src) %{
21426 predicate(VM_Version::supports_avx512vl() || Matcher::vector_element_basic_type(n) != T_DOUBLE);
21427 match(Set dst (VectorCastB2X src));
21428 format %{ "vector_cast_b2x $dst,$src\t!" %}
21429 ins_encode %{
21430 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21431 int vlen_enc = vector_length_encoding(this);
21432 __ vconvert_b2x(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21433 %}
21434 ins_pipe( pipe_slow );
21435 %}
21436
21437 instruct vcastBtoD(legVec dst, legVec src) %{
21438 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_element_basic_type(n) == T_DOUBLE);
21439 match(Set dst (VectorCastB2X src));
21440 format %{ "vector_cast_b2x $dst,$src\t!" %}
21441 ins_encode %{
21442 int vlen_enc = vector_length_encoding(this);
21443 __ vconvert_b2x(T_DOUBLE, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21444 %}
21445 ins_pipe( pipe_slow );
21446 %}
21447
21448 instruct castStoX(vec dst, vec src) %{
21449 predicate((UseAVX <= 2 || !VM_Version::supports_avx512vlbw()) &&
21450 Matcher::vector_length(n->in(1)) <= 8 && // src
21451 Matcher::vector_element_basic_type(n) == T_BYTE);
21452 match(Set dst (VectorCastS2X src));
21453 format %{ "vector_cast_s2x $dst,$src" %}
21454 ins_encode %{
21455 assert(UseAVX > 0, "required");
21456
21457 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), 0, noreg);
21458 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, 0);
21459 %}
21460 ins_pipe( pipe_slow );
21461 %}
21462
21463 instruct vcastStoX(vec dst, vec src, vec vtmp) %{
21464 predicate((UseAVX <= 2 || !VM_Version::supports_avx512vlbw()) &&
21465 Matcher::vector_length(n->in(1)) == 16 && // src
21466 Matcher::vector_element_basic_type(n) == T_BYTE);
21467 effect(TEMP dst, TEMP vtmp);
21468 match(Set dst (VectorCastS2X src));
21469 format %{ "vector_cast_s2x $dst,$src\t! using $vtmp as TEMP" %}
21470 ins_encode %{
21471 assert(UseAVX > 0, "required");
21472
21473 int vlen_enc = vector_length_encoding(Matcher::vector_length_in_bytes(this, $src));
21474 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
21475 __ vextracti128($vtmp$$XMMRegister, $dst$$XMMRegister, 0x1);
21476 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0);
21477 %}
21478 ins_pipe( pipe_slow );
21479 %}
21480
21481 instruct vcastStoX_evex(vec dst, vec src) %{
21482 predicate((UseAVX > 2 && VM_Version::supports_avx512vlbw()) ||
21483 (Matcher::vector_length_in_bytes(n) >= Matcher::vector_length_in_bytes(n->in(1)))); // dst >= src
21484 match(Set dst (VectorCastS2X src));
21485 format %{ "vector_cast_s2x $dst,$src\t!" %}
21486 ins_encode %{
21487 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21488 int src_vlen_enc = vector_length_encoding(this, $src);
21489 int vlen_enc = vector_length_encoding(this);
21490 switch (to_elem_bt) {
21491 case T_BYTE:
21492 if (!VM_Version::supports_avx512vl()) {
21493 vlen_enc = Assembler::AVX_512bit;
21494 }
21495 __ evpmovwb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
21496 break;
21497 case T_INT:
21498 __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21499 break;
21500 case T_FLOAT:
21501 __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21502 __ vcvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21503 break;
21504 case T_LONG:
21505 __ vpmovsxwq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21506 break;
21507 case T_DOUBLE: {
21508 int mid_vlen_enc = (vlen_enc == Assembler::AVX_512bit) ? Assembler::AVX_256bit : Assembler::AVX_128bit;
21509 __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, mid_vlen_enc);
21510 __ vcvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21511 break;
21512 }
21513 default:
21514 ShouldNotReachHere();
21515 }
21516 %}
21517 ins_pipe( pipe_slow );
21518 %}
21519
21520 instruct castItoX(vec dst, vec src) %{
21521 predicate(UseAVX <= 2 &&
21522 (Matcher::vector_length_in_bytes(n->in(1)) <= 16) &&
21523 (Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)))); // dst < src
21524 match(Set dst (VectorCastI2X src));
21525 format %{ "vector_cast_i2x $dst,$src" %}
21526 ins_encode %{
21527 assert(UseAVX > 0, "required");
21528
21529 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21530 int vlen_enc = vector_length_encoding(this, $src);
21531
21532 if (to_elem_bt == T_BYTE) {
21533 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_byte_mask()), vlen_enc, noreg);
21534 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21535 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21536 } else {
21537 assert(to_elem_bt == T_SHORT, "%s", type2name(to_elem_bt));
21538 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21539 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21540 }
21541 %}
21542 ins_pipe( pipe_slow );
21543 %}
21544
21545 instruct vcastItoX(vec dst, vec src, vec vtmp) %{
21546 predicate(UseAVX <= 2 &&
21547 (Matcher::vector_length_in_bytes(n->in(1)) == 32) &&
21548 (Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)))); // dst < src
21549 match(Set dst (VectorCastI2X src));
21550 format %{ "vector_cast_i2x $dst,$src\t! using $vtmp as TEMP" %}
21551 effect(TEMP dst, TEMP vtmp);
21552 ins_encode %{
21553 assert(UseAVX > 0, "required");
21554
21555 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21556 int vlen_enc = vector_length_encoding(this, $src);
21557
21558 if (to_elem_bt == T_BYTE) {
21559 __ vpand($vtmp$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_byte_mask()), vlen_enc, noreg);
21560 __ vextracti128($dst$$XMMRegister, $vtmp$$XMMRegister, 0x1);
21561 __ vpackusdw($dst$$XMMRegister, $vtmp$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21562 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21563 } else {
21564 assert(to_elem_bt == T_SHORT, "%s", type2name(to_elem_bt));
21565 __ vpand($vtmp$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21566 __ vextracti128($dst$$XMMRegister, $vtmp$$XMMRegister, 0x1);
21567 __ vpackusdw($dst$$XMMRegister, $vtmp$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21568 }
21569 %}
21570 ins_pipe( pipe_slow );
21571 %}
21572
21573 instruct vcastItoX_evex(vec dst, vec src) %{
21574 predicate(UseAVX > 2 ||
21575 (Matcher::vector_length_in_bytes(n) >= Matcher::vector_length_in_bytes(n->in(1)))); // dst >= src
21576 match(Set dst (VectorCastI2X src));
21577 format %{ "vector_cast_i2x $dst,$src\t!" %}
21578 ins_encode %{
21579 assert(UseAVX > 0, "required");
21580
21581 BasicType dst_elem_bt = Matcher::vector_element_basic_type(this);
21582 int src_vlen_enc = vector_length_encoding(this, $src);
21583 int dst_vlen_enc = vector_length_encoding(this);
21584 switch (dst_elem_bt) {
21585 case T_BYTE:
21586 if (!VM_Version::supports_avx512vl()) {
21587 src_vlen_enc = Assembler::AVX_512bit;
21588 }
21589 __ evpmovdb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
21590 break;
21591 case T_SHORT:
21592 if (!VM_Version::supports_avx512vl()) {
21593 src_vlen_enc = Assembler::AVX_512bit;
21594 }
21595 __ evpmovdw($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
21596 break;
21597 case T_FLOAT:
21598 __ vcvtdq2ps($dst$$XMMRegister, $src$$XMMRegister, dst_vlen_enc);
21599 break;
21600 case T_LONG:
21601 __ vpmovsxdq($dst$$XMMRegister, $src$$XMMRegister, dst_vlen_enc);
21602 break;
21603 case T_DOUBLE:
21604 __ vcvtdq2pd($dst$$XMMRegister, $src$$XMMRegister, dst_vlen_enc);
21605 break;
21606 default:
21607 ShouldNotReachHere();
21608 }
21609 %}
21610 ins_pipe( pipe_slow );
21611 %}
21612
21613 instruct vcastLtoBS(vec dst, vec src) %{
21614 predicate((Matcher::vector_element_basic_type(n) == T_BYTE || Matcher::vector_element_basic_type(n) == T_SHORT) &&
21615 UseAVX <= 2);
21616 match(Set dst (VectorCastL2X src));
21617 format %{ "vector_cast_l2x $dst,$src" %}
21618 ins_encode %{
21619 assert(UseAVX > 0, "required");
21620
21621 int vlen = Matcher::vector_length_in_bytes(this, $src);
21622 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21623 AddressLiteral mask_addr = (to_elem_bt == T_BYTE) ? ExternalAddress(vector_int_to_byte_mask())
21624 : ExternalAddress(vector_int_to_short_mask());
21625 if (vlen <= 16) {
21626 __ vpshufd($dst$$XMMRegister, $src$$XMMRegister, 8, Assembler::AVX_128bit);
21627 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, mask_addr, Assembler::AVX_128bit, noreg);
21628 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21629 } else {
21630 assert(vlen <= 32, "required");
21631 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 8, Assembler::AVX_256bit);
21632 __ vpermpd($dst$$XMMRegister, $dst$$XMMRegister, 8, Assembler::AVX_256bit);
21633 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, mask_addr, Assembler::AVX_128bit, noreg);
21634 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21635 }
21636 if (to_elem_bt == T_BYTE) {
21637 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21638 }
21639 %}
21640 ins_pipe( pipe_slow );
21641 %}
21642
21643 instruct vcastLtoX_evex(vec dst, vec src) %{
21644 predicate(UseAVX > 2 ||
21645 (Matcher::vector_element_basic_type(n) == T_INT ||
21646 Matcher::vector_element_basic_type(n) == T_FLOAT ||
21647 Matcher::vector_element_basic_type(n) == T_DOUBLE));
21648 match(Set dst (VectorCastL2X src));
21649 format %{ "vector_cast_l2x $dst,$src\t!" %}
21650 ins_encode %{
21651 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21652 int vlen = Matcher::vector_length_in_bytes(this, $src);
21653 int vlen_enc = vector_length_encoding(this, $src);
21654 switch (to_elem_bt) {
21655 case T_BYTE:
21656 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
21657 vlen_enc = Assembler::AVX_512bit;
21658 }
21659 __ evpmovqb($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21660 break;
21661 case T_SHORT:
21662 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
21663 vlen_enc = Assembler::AVX_512bit;
21664 }
21665 __ evpmovqw($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21666 break;
21667 case T_INT:
21668 if (vlen == 8) {
21669 if ($dst$$XMMRegister != $src$$XMMRegister) {
21670 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
21671 }
21672 } else if (vlen == 16) {
21673 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 8);
21674 } else if (vlen == 32) {
21675 if (UseAVX > 2) {
21676 if (!VM_Version::supports_avx512vl()) {
21677 vlen_enc = Assembler::AVX_512bit;
21678 }
21679 __ evpmovqd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21680 } else {
21681 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 8, vlen_enc);
21682 __ vpermpd($dst$$XMMRegister, $dst$$XMMRegister, 8, vlen_enc);
21683 }
21684 } else { // vlen == 64
21685 __ evpmovqd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21686 }
21687 break;
21688 case T_FLOAT:
21689 assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "required");
21690 __ evcvtqq2ps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21691 break;
21692 case T_DOUBLE:
21693 assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "required");
21694 __ evcvtqq2pd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21695 break;
21696
21697 default: assert(false, "%s", type2name(to_elem_bt));
21698 }
21699 %}
21700 ins_pipe( pipe_slow );
21701 %}
21702
21703 instruct vcastFtoD_reg(vec dst, vec src) %{
21704 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
21705 match(Set dst (VectorCastF2X src));
21706 format %{ "vector_cast_f2d $dst,$src\t!" %}
21707 ins_encode %{
21708 int vlen_enc = vector_length_encoding(this);
21709 __ vcvtps2pd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21710 %}
21711 ins_pipe( pipe_slow );
21712 %}
21713
21714
21715 instruct castFtoX_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, rFlagsReg cr) %{
21716 predicate(!VM_Version::supports_avx10_2() &&
21717 !VM_Version::supports_avx512vl() &&
21718 Matcher::vector_length_in_bytes(n->in(1)) < 64 &&
21719 type2aelembytes(Matcher::vector_element_basic_type(n)) <= 4 &&
21720 is_integral_type(Matcher::vector_element_basic_type(n)));
21721 match(Set dst (VectorCastF2X src));
21722 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, KILL cr);
21723 format %{ "vector_cast_f2x $dst,$src\t! using $xtmp1, $xtmp2, $xtmp3 and $xtmp4 as TEMP" %}
21724 ins_encode %{
21725 int vlen_enc = vector_length_encoding(this, $src);
21726 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21727 // JDK-8292878 removed the need for an explicit scratch register needed to load greater than
21728 // 32 bit addresses for register indirect addressing mode since stub constants
21729 // are part of code cache and there is a cap of 2G on ReservedCodeCacheSize currently.
21730 // However, targets are free to increase this limit, but having a large code cache size
21731 // greater than 2G looks unreasonable in practical scenario, on the hind side with given
21732 // cap we save a temporary register allocation which in limiting case can prevent
21733 // spilling in high register pressure blocks.
21734 __ vector_castF2X_avx(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
21735 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister,
21736 ExternalAddress(vector_float_signflip()), noreg, vlen_enc);
21737 %}
21738 ins_pipe( pipe_slow );
21739 %}
21740
21741 instruct castFtoX_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
21742 predicate(!VM_Version::supports_avx10_2() &&
21743 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n->in(1)) == 64) &&
21744 is_integral_type(Matcher::vector_element_basic_type(n)));
21745 match(Set dst (VectorCastF2X src));
21746 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
21747 format %{ "vector_cast_f2x $dst,$src\t! using $xtmp1, $xtmp2, $ktmp1 and $ktmp2 as TEMP" %}
21748 ins_encode %{
21749 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21750 if (to_elem_bt == T_LONG) {
21751 int vlen_enc = vector_length_encoding(this);
21752 __ vector_castF2L_evex($dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
21753 $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister,
21754 ExternalAddress(vector_double_signflip()), noreg, vlen_enc);
21755 } else {
21756 int vlen_enc = vector_length_encoding(this, $src);
21757 __ vector_castF2X_evex(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
21758 $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister,
21759 ExternalAddress(vector_float_signflip()), noreg, vlen_enc);
21760 }
21761 %}
21762 ins_pipe( pipe_slow );
21763 %}
21764
21765 instruct castFtoX_reg_avx10_2(vec dst, vec src) %{
21766 predicate(VM_Version::supports_avx10_2() &&
21767 is_integral_type(Matcher::vector_element_basic_type(n)));
21768 match(Set dst (VectorCastF2X src));
21769 format %{ "vector_cast_f2x_avx10_2 $dst, $src\t!" %}
21770 ins_encode %{
21771 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21772 int vlen_enc = (to_elem_bt == T_LONG) ? vector_length_encoding(this) : vector_length_encoding(this, $src);
21773 __ vector_castF2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21774 %}
21775 ins_pipe( pipe_slow );
21776 %}
21777
21778 instruct castFtoX_mem_avx10_2(vec dst, memory src) %{
21779 predicate(VM_Version::supports_avx10_2() &&
21780 is_integral_type(Matcher::vector_element_basic_type(n)));
21781 match(Set dst (VectorCastF2X (LoadVector src)));
21782 format %{ "vector_cast_f2x_avx10_2 $dst, $src\t!" %}
21783 ins_encode %{
21784 int vlen = Matcher::vector_length(this);
21785 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21786 int vlen_enc = (to_elem_bt == T_LONG) ? vector_length_encoding(this) : vector_length_encoding(vlen * sizeof(jfloat));
21787 __ vector_castF2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$Address, vlen_enc);
21788 %}
21789 ins_pipe( pipe_slow );
21790 %}
21791
21792 instruct vcastDtoF_reg(vec dst, vec src) %{
21793 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
21794 match(Set dst (VectorCastD2X src));
21795 format %{ "vector_cast_d2x $dst,$src\t!" %}
21796 ins_encode %{
21797 int vlen_enc = vector_length_encoding(this, $src);
21798 __ vcvtpd2ps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21799 %}
21800 ins_pipe( pipe_slow );
21801 %}
21802
21803 instruct castDtoX_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, vec xtmp5, rFlagsReg cr) %{
21804 predicate(!VM_Version::supports_avx10_2() &&
21805 !VM_Version::supports_avx512vl() &&
21806 Matcher::vector_length_in_bytes(n->in(1)) < 64 &&
21807 is_integral_type(Matcher::vector_element_basic_type(n)));
21808 match(Set dst (VectorCastD2X src));
21809 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, TEMP xtmp5, KILL cr);
21810 format %{ "vector_cast_d2x $dst,$src\t! using $xtmp1, $xtmp2, $xtmp3, $xtmp4 and $xtmp5 as TEMP" %}
21811 ins_encode %{
21812 int vlen_enc = vector_length_encoding(this, $src);
21813 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21814 __ vector_castD2X_avx(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
21815 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister, $xtmp5$$XMMRegister,
21816 ExternalAddress(vector_float_signflip()), noreg, vlen_enc);
21817 %}
21818 ins_pipe( pipe_slow );
21819 %}
21820
21821 instruct castDtoX_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
21822 predicate(!VM_Version::supports_avx10_2() &&
21823 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n->in(1)) == 64) &&
21824 is_integral_type(Matcher::vector_element_basic_type(n)));
21825 match(Set dst (VectorCastD2X src));
21826 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
21827 format %{ "vector_cast_d2x $dst,$src\t! using $xtmp1, $xtmp2, $ktmp1 and $ktmp2 as TEMP" %}
21828 ins_encode %{
21829 int vlen_enc = vector_length_encoding(this, $src);
21830 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21831 AddressLiteral signflip = VM_Version::supports_avx512dq() ? ExternalAddress(vector_double_signflip()) :
21832 ExternalAddress(vector_float_signflip());
21833 __ vector_castD2X_evex(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
21834 $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister, signflip, noreg, vlen_enc);
21835 %}
21836 ins_pipe( pipe_slow );
21837 %}
21838
21839 instruct castDtoX_reg_avx10_2(vec dst, vec src) %{
21840 predicate(VM_Version::supports_avx10_2() &&
21841 is_integral_type(Matcher::vector_element_basic_type(n)));
21842 match(Set dst (VectorCastD2X src));
21843 format %{ "vector_cast_d2x_avx10_2 $dst, $src\t!" %}
21844 ins_encode %{
21845 int vlen_enc = vector_length_encoding(this, $src);
21846 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21847 __ vector_castD2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21848 %}
21849 ins_pipe( pipe_slow );
21850 %}
21851
21852 instruct castDtoX_mem_avx10_2(vec dst, memory src) %{
21853 predicate(VM_Version::supports_avx10_2() &&
21854 is_integral_type(Matcher::vector_element_basic_type(n)));
21855 match(Set dst (VectorCastD2X (LoadVector src)));
21856 format %{ "vector_cast_d2x_avx10_2 $dst, $src\t!" %}
21857 ins_encode %{
21858 int vlen = Matcher::vector_length(this);
21859 int vlen_enc = vector_length_encoding(vlen * sizeof(jdouble));
21860 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21861 __ vector_castD2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$Address, vlen_enc);
21862 %}
21863 ins_pipe( pipe_slow );
21864 %}
21865
21866 instruct vucast(vec dst, vec src) %{
21867 match(Set dst (VectorUCastB2X src));
21868 match(Set dst (VectorUCastS2X src));
21869 match(Set dst (VectorUCastI2X src));
21870 format %{ "vector_ucast $dst,$src\t!" %}
21871 ins_encode %{
21872 assert(UseAVX > 0, "required");
21873
21874 BasicType from_elem_bt = Matcher::vector_element_basic_type(this, $src);
21875 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21876 int vlen_enc = vector_length_encoding(this);
21877 __ vector_unsigned_cast($dst$$XMMRegister, $src$$XMMRegister, vlen_enc, from_elem_bt, to_elem_bt);
21878 %}
21879 ins_pipe( pipe_slow );
21880 %}
21881
21882 instruct vround_float_avx(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, rFlagsReg cr) %{
21883 predicate(!VM_Version::supports_avx512vl() &&
21884 Matcher::vector_length_in_bytes(n) < 64 &&
21885 Matcher::vector_element_basic_type(n) == T_INT);
21886 match(Set dst (RoundVF src));
21887 effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, KILL cr);
21888 format %{ "vector_round_float $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $xtmp3, $xtmp4 as TEMP" %}
21889 ins_encode %{
21890 int vlen_enc = vector_length_encoding(this);
21891 InternalAddress new_mxcsr = $constantaddress((jint)(EnableX86ECoreOpts ? 0x3FBF : 0x3F80));
21892 __ vector_round_float_avx($dst$$XMMRegister, $src$$XMMRegister,
21893 ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), new_mxcsr, vlen_enc,
21894 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister);
21895 %}
21896 ins_pipe( pipe_slow );
21897 %}
21898
21899 instruct vround_float_evex(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
21900 predicate((VM_Version::supports_avx512vl() ||
21901 Matcher::vector_length_in_bytes(n) == 64) &&
21902 Matcher::vector_element_basic_type(n) == T_INT);
21903 match(Set dst (RoundVF src));
21904 effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
21905 format %{ "vector_round_float $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $ktmp1, $ktmp2 as TEMP" %}
21906 ins_encode %{
21907 int vlen_enc = vector_length_encoding(this);
21908 InternalAddress new_mxcsr = $constantaddress((jint)(EnableX86ECoreOpts ? 0x3FBF : 0x3F80));
21909 __ vector_round_float_evex($dst$$XMMRegister, $src$$XMMRegister,
21910 ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), new_mxcsr, vlen_enc,
21911 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
21912 %}
21913 ins_pipe( pipe_slow );
21914 %}
21915
21916 instruct vround_reg_evex(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
21917 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
21918 match(Set dst (RoundVD src));
21919 effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
21920 format %{ "vector_round_long $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $ktmp1, $ktmp2 as TEMP" %}
21921 ins_encode %{
21922 int vlen_enc = vector_length_encoding(this);
21923 InternalAddress new_mxcsr = $constantaddress((jint)(EnableX86ECoreOpts ? 0x3FBF : 0x3F80));
21924 __ vector_round_double_evex($dst$$XMMRegister, $src$$XMMRegister,
21925 ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), new_mxcsr, vlen_enc,
21926 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
21927 %}
21928 ins_pipe( pipe_slow );
21929 %}
21930
21931 // --------------------------------- VectorMaskCmp --------------------------------------
21932
21933 instruct vcmpFD(legVec dst, legVec src1, legVec src2, immI8 cond) %{
21934 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
21935 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 8 && // src1
21936 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
21937 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
21938 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
21939 format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
21940 ins_encode %{
21941 int vlen_enc = vector_length_encoding(this, $src1);
21942 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
21943 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
21944 __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
21945 } else {
21946 __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
21947 }
21948 %}
21949 ins_pipe( pipe_slow );
21950 %}
21951
21952 instruct evcmpFD64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
21953 predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64 && // src1
21954 n->bottom_type()->isa_pvectmask() == nullptr &&
21955 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
21956 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
21957 effect(TEMP ktmp);
21958 format %{ "vector_compare $dst,$src1,$src2,$cond" %}
21959 ins_encode %{
21960 int vlen_enc = Assembler::AVX_512bit;
21961 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
21962 KRegister mask = k0; // The comparison itself is not being masked.
21963 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
21964 __ evcmpps($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
21965 __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, noreg);
21966 } else {
21967 __ evcmppd($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
21968 __ evmovdquq($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, noreg);
21969 }
21970 %}
21971 ins_pipe( pipe_slow );
21972 %}
21973
21974 instruct evcmpFD(kReg dst, vec src1, vec src2, immI8 cond) %{
21975 predicate(n->bottom_type()->isa_pvectmask() &&
21976 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
21977 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
21978 format %{ "vector_compare_evex $dst,$src1,$src2,$cond\t!" %}
21979 ins_encode %{
21980 assert(bottom_type()->isa_pvectmask(), "TypePVectMask expected");
21981 int vlen_enc = vector_length_encoding(this, $src1);
21982 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
21983 KRegister mask = k0; // The comparison itself is not being masked.
21984 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
21985 __ evcmpps($dst$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
21986 } else {
21987 __ evcmppd($dst$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
21988 }
21989 %}
21990 ins_pipe( pipe_slow );
21991 %}
21992
21993 instruct vcmp_direct(legVec dst, legVec src1, legVec src2, immI8 cond) %{
21994 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
21995 !Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
21996 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
21997 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
21998 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1))) &&
21999 (n->in(2)->get_int() == BoolTest::eq ||
22000 n->in(2)->get_int() == BoolTest::lt ||
22001 n->in(2)->get_int() == BoolTest::gt)); // cond
22002 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22003 format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
22004 ins_encode %{
22005 int vlen_enc = vector_length_encoding(this, $src1);
22006 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22007 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
22008 __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, cmp, ww, vlen_enc);
22009 %}
22010 ins_pipe( pipe_slow );
22011 %}
22012
22013 instruct vcmp_negate(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
22014 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
22015 !Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
22016 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
22017 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
22018 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1))) &&
22019 (n->in(2)->get_int() == BoolTest::ne ||
22020 n->in(2)->get_int() == BoolTest::le ||
22021 n->in(2)->get_int() == BoolTest::ge)); // cond
22022 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22023 effect(TEMP dst, TEMP xtmp);
22024 format %{ "vector_compare $dst,$src1,$src2,$cond\t! using $xtmp as TEMP" %}
22025 ins_encode %{
22026 int vlen_enc = vector_length_encoding(this, $src1);
22027 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22028 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
22029 __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $xtmp$$XMMRegister, cmp, ww, vlen_enc);
22030 %}
22031 ins_pipe( pipe_slow );
22032 %}
22033
22034 instruct vcmpu(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
22035 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
22036 Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
22037 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
22038 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
22039 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
22040 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22041 effect(TEMP dst, TEMP xtmp);
22042 format %{ "vector_compareu $dst,$src1,$src2,$cond\t! using $xtmp as TEMP" %}
22043 ins_encode %{
22044 InternalAddress flip_bit = $constantaddress(high_bit_set(Matcher::vector_element_basic_type(this, $src1)));
22045 int vlen_enc = vector_length_encoding(this, $src1);
22046 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22047 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
22048
22049 if (vlen_enc == Assembler::AVX_128bit) {
22050 __ vmovddup($xtmp$$XMMRegister, flip_bit, vlen_enc, noreg);
22051 } else {
22052 __ vbroadcastsd($xtmp$$XMMRegister, flip_bit, vlen_enc, noreg);
22053 }
22054 __ vpxor($dst$$XMMRegister, $xtmp$$XMMRegister, $src1$$XMMRegister, vlen_enc);
22055 __ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $src2$$XMMRegister, vlen_enc);
22056 __ vpcmpCCW($dst$$XMMRegister, $dst$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, cmp, ww, vlen_enc);
22057 %}
22058 ins_pipe( pipe_slow );
22059 %}
22060
22061 instruct vcmp64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
22062 predicate((n->bottom_type()->isa_pvectmask() == nullptr &&
22063 Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64) && // src1
22064 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
22065 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22066 effect(TEMP ktmp);
22067 format %{ "vector_compare $dst,$src1,$src2,$cond" %}
22068 ins_encode %{
22069 assert(UseAVX > 2, "required");
22070
22071 int vlen_enc = vector_length_encoding(this, $src1);
22072 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22073 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
22074 KRegister mask = k0; // The comparison itself is not being masked.
22075 bool merge = false;
22076 BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
22077
22078 switch (src1_elem_bt) {
22079 case T_INT: {
22080 __ evpcmpd($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22081 __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), merge, vlen_enc, noreg);
22082 break;
22083 }
22084 case T_LONG: {
22085 __ evpcmpq($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22086 __ evmovdquq($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), merge, vlen_enc, noreg);
22087 break;
22088 }
22089 default: assert(false, "%s", type2name(src1_elem_bt));
22090 }
22091 %}
22092 ins_pipe( pipe_slow );
22093 %}
22094
22095
22096 instruct evcmp(kReg dst, vec src1, vec src2, immI8 cond) %{
22097 predicate(n->bottom_type()->isa_pvectmask() &&
22098 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
22099 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22100 format %{ "vector_compared_evex $dst,$src1,$src2,$cond\t!" %}
22101 ins_encode %{
22102 assert(UseAVX > 2, "required");
22103 assert(bottom_type()->isa_pvectmask(), "TypePVectMask expected");
22104
22105 int vlen_enc = vector_length_encoding(this, $src1);
22106 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22107 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
22108 BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
22109
22110 // Comparison i
22111 switch (src1_elem_bt) {
22112 case T_BYTE: {
22113 __ evpcmpb($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22114 break;
22115 }
22116 case T_SHORT: {
22117 __ evpcmpw($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22118 break;
22119 }
22120 case T_INT: {
22121 __ evpcmpd($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22122 break;
22123 }
22124 case T_LONG: {
22125 __ evpcmpq($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22126 break;
22127 }
22128 default: assert(false, "%s", type2name(src1_elem_bt));
22129 }
22130 %}
22131 ins_pipe( pipe_slow );
22132 %}
22133
22134 // Extract
22135
22136 instruct extractI(rRegI dst, legVec src, immU8 idx) %{
22137 predicate(Matcher::vector_length_in_bytes(n->in(1)) <= 16); // src
22138 match(Set dst (ExtractI src idx));
22139 match(Set dst (ExtractS src idx));
22140 match(Set dst (ExtractB src idx));
22141 format %{ "extractI $dst,$src,$idx\t!" %}
22142 ins_encode %{
22143 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22144
22145 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
22146 __ get_elem(elem_bt, $dst$$Register, $src$$XMMRegister, $idx$$constant);
22147 %}
22148 ins_pipe( pipe_slow );
22149 %}
22150
22151 instruct vextractI(rRegI dst, legVec src, immI idx, legVec vtmp) %{
22152 predicate(Matcher::vector_length_in_bytes(n->in(1)) == 32 || // src
22153 Matcher::vector_length_in_bytes(n->in(1)) == 64); // src
22154 match(Set dst (ExtractI src idx));
22155 match(Set dst (ExtractS src idx));
22156 match(Set dst (ExtractB src idx));
22157 effect(TEMP vtmp);
22158 format %{ "vextractI $dst,$src,$idx\t! using $vtmp as TEMP" %}
22159 ins_encode %{
22160 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22161
22162 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
22163 XMMRegister lane_xmm = __ get_lane(elem_bt, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22164 __ get_elem(elem_bt, $dst$$Register, lane_xmm, $idx$$constant);
22165 %}
22166 ins_pipe( pipe_slow );
22167 %}
22168
22169 instruct extractL(rRegL dst, legVec src, immU8 idx) %{
22170 predicate(Matcher::vector_length(n->in(1)) <= 2); // src
22171 match(Set dst (ExtractL src idx));
22172 format %{ "extractL $dst,$src,$idx\t!" %}
22173 ins_encode %{
22174 assert(UseSSE >= 4, "required");
22175 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22176
22177 __ get_elem(T_LONG, $dst$$Register, $src$$XMMRegister, $idx$$constant);
22178 %}
22179 ins_pipe( pipe_slow );
22180 %}
22181
22182 instruct vextractL(rRegL dst, legVec src, immU8 idx, legVec vtmp) %{
22183 predicate(Matcher::vector_length(n->in(1)) == 4 || // src
22184 Matcher::vector_length(n->in(1)) == 8); // src
22185 match(Set dst (ExtractL src idx));
22186 effect(TEMP vtmp);
22187 format %{ "vextractL $dst,$src,$idx\t! using $vtmp as TEMP" %}
22188 ins_encode %{
22189 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22190
22191 XMMRegister lane_reg = __ get_lane(T_LONG, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22192 __ get_elem(T_LONG, $dst$$Register, lane_reg, $idx$$constant);
22193 %}
22194 ins_pipe( pipe_slow );
22195 %}
22196
22197 instruct extractF(legRegF dst, legVec src, immU8 idx, legVec vtmp) %{
22198 predicate(Matcher::vector_length(n->in(1)) <= 4);
22199 match(Set dst (ExtractF src idx));
22200 effect(TEMP dst, TEMP vtmp);
22201 format %{ "extractF $dst,$src,$idx\t! using $vtmp as TEMP" %}
22202 ins_encode %{
22203 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22204
22205 __ get_elem(T_FLOAT, $dst$$XMMRegister, $src$$XMMRegister, $idx$$constant, $vtmp$$XMMRegister);
22206 %}
22207 ins_pipe( pipe_slow );
22208 %}
22209
22210 instruct vextractF(legRegF dst, legVec src, immU8 idx, legVec vtmp) %{
22211 predicate(Matcher::vector_length(n->in(1)/*src*/) == 8 ||
22212 Matcher::vector_length(n->in(1)/*src*/) == 16);
22213 match(Set dst (ExtractF src idx));
22214 effect(TEMP vtmp);
22215 format %{ "vextractF $dst,$src,$idx\t! using $vtmp as TEMP" %}
22216 ins_encode %{
22217 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22218
22219 XMMRegister lane_reg = __ get_lane(T_FLOAT, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22220 __ get_elem(T_FLOAT, $dst$$XMMRegister, lane_reg, $idx$$constant);
22221 %}
22222 ins_pipe( pipe_slow );
22223 %}
22224
22225 instruct extractD(legRegD dst, legVec src, immU8 idx) %{
22226 predicate(Matcher::vector_length(n->in(1)) == 2); // src
22227 match(Set dst (ExtractD src idx));
22228 format %{ "extractD $dst,$src,$idx\t!" %}
22229 ins_encode %{
22230 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22231
22232 __ get_elem(T_DOUBLE, $dst$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22233 %}
22234 ins_pipe( pipe_slow );
22235 %}
22236
22237 instruct vextractD(legRegD dst, legVec src, immU8 idx, legVec vtmp) %{
22238 predicate(Matcher::vector_length(n->in(1)) == 4 || // src
22239 Matcher::vector_length(n->in(1)) == 8); // src
22240 match(Set dst (ExtractD src idx));
22241 effect(TEMP vtmp);
22242 format %{ "vextractD $dst,$src,$idx\t! using $vtmp as TEMP" %}
22243 ins_encode %{
22244 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22245
22246 XMMRegister lane_reg = __ get_lane(T_DOUBLE, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22247 __ get_elem(T_DOUBLE, $dst$$XMMRegister, lane_reg, $idx$$constant);
22248 %}
22249 ins_pipe( pipe_slow );
22250 %}
22251
22252 // --------------------------------- Vector Blend --------------------------------------
22253
22254 instruct blendvp(vec dst, vec src, vec mask, rxmm0 tmp) %{
22255 predicate(UseAVX == 0);
22256 match(Set dst (VectorBlend (Binary dst src) mask));
22257 format %{ "vector_blend $dst,$src,$mask\t! using $tmp as TEMP" %}
22258 effect(TEMP tmp);
22259 ins_encode %{
22260 assert(UseSSE >= 4, "required");
22261
22262 if ($mask$$XMMRegister != $tmp$$XMMRegister) {
22263 __ movdqu($tmp$$XMMRegister, $mask$$XMMRegister);
22264 }
22265 __ pblendvb($dst$$XMMRegister, $src$$XMMRegister); // uses xmm0 as mask
22266 %}
22267 ins_pipe( pipe_slow );
22268 %}
22269
22270 instruct vblendvpI(legVec dst, legVec src1, legVec src2, legVec mask) %{
22271 predicate(UseAVX > 0 && !EnableX86ECoreOpts &&
22272 n->in(2)->bottom_type()->isa_pvectmask() == nullptr &&
22273 Matcher::vector_length_in_bytes(n) <= 32 &&
22274 is_integral_type(Matcher::vector_element_basic_type(n)));
22275 match(Set dst (VectorBlend (Binary src1 src2) mask));
22276 format %{ "vector_blend $dst,$src1,$src2,$mask\t!" %}
22277 ins_encode %{
22278 int vlen_enc = vector_length_encoding(this);
22279 __ vpblendvb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $mask$$XMMRegister, vlen_enc);
22280 %}
22281 ins_pipe( pipe_slow );
22282 %}
22283
22284 instruct vblendvpFD(legVec dst, legVec src1, legVec src2, legVec mask) %{
22285 predicate(UseAVX > 0 && !EnableX86ECoreOpts &&
22286 n->in(2)->bottom_type()->isa_pvectmask() == nullptr &&
22287 Matcher::vector_length_in_bytes(n) <= 32 &&
22288 !is_integral_type(Matcher::vector_element_basic_type(n)));
22289 match(Set dst (VectorBlend (Binary src1 src2) mask));
22290 format %{ "vector_blend $dst,$src1,$src2,$mask\t!" %}
22291 ins_encode %{
22292 int vlen_enc = vector_length_encoding(this);
22293 __ vblendvps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $mask$$XMMRegister, vlen_enc);
22294 %}
22295 ins_pipe( pipe_slow );
22296 %}
22297
22298 instruct vblendvp(legVec dst, legVec src1, legVec src2, legVec mask, legVec vtmp) %{
22299 predicate(UseAVX > 0 && EnableX86ECoreOpts &&
22300 n->in(2)->bottom_type()->isa_pvectmask() == nullptr &&
22301 Matcher::vector_length_in_bytes(n) <= 32);
22302 match(Set dst (VectorBlend (Binary src1 src2) mask));
22303 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using $vtmp as TEMP" %}
22304 effect(TEMP vtmp, TEMP dst);
22305 ins_encode %{
22306 int vlen_enc = vector_length_encoding(this);
22307 __ vpandn($vtmp$$XMMRegister, $mask$$XMMRegister, $src1$$XMMRegister, vlen_enc);
22308 __ vpand ($dst$$XMMRegister, $mask$$XMMRegister, $src2$$XMMRegister, vlen_enc);
22309 __ vpor ($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22310 %}
22311 ins_pipe( pipe_slow );
22312 %}
22313
22314 instruct evblendvp64(vec dst, vec src1, vec src2, vec mask, kReg ktmp) %{
22315 predicate(Matcher::vector_length_in_bytes(n) == 64 &&
22316 n->in(2)->bottom_type()->isa_pvectmask() == nullptr);
22317 match(Set dst (VectorBlend (Binary src1 src2) mask));
22318 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
22319 effect(TEMP ktmp);
22320 ins_encode %{
22321 int vlen_enc = Assembler::AVX_512bit;
22322 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22323 __ evpcmp(elem_bt, $ktmp$$KRegister, k0, $mask$$XMMRegister, ExternalAddress(vector_all_bits_set()), Assembler::eq, vlen_enc, noreg);
22324 __ evpblend(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
22325 %}
22326 ins_pipe( pipe_slow );
22327 %}
22328
22329
22330 instruct evblendvp64_masked(vec dst, vec src1, vec src2, kReg mask) %{
22331 predicate(n->in(2)->bottom_type()->isa_pvectmask() &&
22332 (!is_subword_type(Matcher::vector_element_basic_type(n)) ||
22333 VM_Version::supports_avx512bw()));
22334 match(Set dst (VectorBlend (Binary src1 src2) mask));
22335 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
22336 ins_encode %{
22337 int vlen_enc = vector_length_encoding(this);
22338 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22339 __ evpblend(elem_bt, $dst$$XMMRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
22340 %}
22341 ins_pipe( pipe_slow );
22342 %}
22343
22344 // --------------------------------- ABS --------------------------------------
22345 // a = |a|
22346 instruct vabsB_reg(vec dst, vec src) %{
22347 match(Set dst (AbsVB src));
22348 format %{ "vabsb $dst,$src\t# $dst = |$src| abs packedB" %}
22349 ins_encode %{
22350 uint vlen = Matcher::vector_length(this);
22351 if (vlen <= 16) {
22352 __ pabsb($dst$$XMMRegister, $src$$XMMRegister);
22353 } else {
22354 int vlen_enc = vector_length_encoding(this);
22355 __ vpabsb($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22356 }
22357 %}
22358 ins_pipe( pipe_slow );
22359 %}
22360
22361 instruct vabsS_reg(vec dst, vec src) %{
22362 match(Set dst (AbsVS src));
22363 format %{ "vabsw $dst,$src\t# $dst = |$src| abs packedS" %}
22364 ins_encode %{
22365 uint vlen = Matcher::vector_length(this);
22366 if (vlen <= 8) {
22367 __ pabsw($dst$$XMMRegister, $src$$XMMRegister);
22368 } else {
22369 int vlen_enc = vector_length_encoding(this);
22370 __ vpabsw($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22371 }
22372 %}
22373 ins_pipe( pipe_slow );
22374 %}
22375
22376 instruct vabsI_reg(vec dst, vec src) %{
22377 match(Set dst (AbsVI src));
22378 format %{ "pabsd $dst,$src\t# $dst = |$src| abs packedI" %}
22379 ins_encode %{
22380 uint vlen = Matcher::vector_length(this);
22381 if (vlen <= 4) {
22382 __ pabsd($dst$$XMMRegister, $src$$XMMRegister);
22383 } else {
22384 int vlen_enc = vector_length_encoding(this);
22385 __ vpabsd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22386 }
22387 %}
22388 ins_pipe( pipe_slow );
22389 %}
22390
22391 instruct vabsL_reg(vec dst, vec src) %{
22392 match(Set dst (AbsVL src));
22393 format %{ "evpabsq $dst,$src\t# $dst = |$src| abs packedL" %}
22394 ins_encode %{
22395 assert(UseAVX > 2, "required");
22396 int vlen_enc = vector_length_encoding(this);
22397 if (!VM_Version::supports_avx512vl()) {
22398 vlen_enc = Assembler::AVX_512bit;
22399 }
22400 __ evpabsq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22401 %}
22402 ins_pipe( pipe_slow );
22403 %}
22404
22405 // --------------------------------- ABSNEG --------------------------------------
22406
22407 instruct vabsnegF(vec dst, vec src) %{
22408 predicate(Matcher::vector_length(n) != 4); // handled by 1-operand instruction vabsneg4F
22409 match(Set dst (AbsVF src));
22410 match(Set dst (NegVF src));
22411 format %{ "vabsnegf $dst,$src,[mask]\t# absneg packedF" %}
22412 ins_cost(150);
22413 ins_encode %{
22414 int opcode = this->ideal_Opcode();
22415 int vlen = Matcher::vector_length(this);
22416 if (vlen == 2) {
22417 __ vabsnegf(opcode, $dst$$XMMRegister, $src$$XMMRegister);
22418 } else {
22419 assert(vlen == 8 || vlen == 16, "required");
22420 int vlen_enc = vector_length_encoding(this);
22421 __ vabsnegf(opcode, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22422 }
22423 %}
22424 ins_pipe( pipe_slow );
22425 %}
22426
22427 instruct vabsneg4F(vec dst) %{
22428 predicate(Matcher::vector_length(n) == 4);
22429 match(Set dst (AbsVF dst));
22430 match(Set dst (NegVF dst));
22431 format %{ "vabsnegf $dst,[mask]\t# absneg packed4F" %}
22432 ins_cost(150);
22433 ins_encode %{
22434 int opcode = this->ideal_Opcode();
22435 __ vabsnegf(opcode, $dst$$XMMRegister, $dst$$XMMRegister);
22436 %}
22437 ins_pipe( pipe_slow );
22438 %}
22439
22440 instruct vabsnegD(vec dst, vec src) %{
22441 match(Set dst (AbsVD src));
22442 match(Set dst (NegVD src));
22443 format %{ "vabsnegd $dst,$src,[mask]\t# absneg packedD" %}
22444 ins_encode %{
22445 int opcode = this->ideal_Opcode();
22446 uint vlen = Matcher::vector_length(this);
22447 if (vlen == 2) {
22448 __ vabsnegd(opcode, $dst$$XMMRegister, $src$$XMMRegister);
22449 } else {
22450 int vlen_enc = vector_length_encoding(this);
22451 __ vabsnegd(opcode, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22452 }
22453 %}
22454 ins_pipe( pipe_slow );
22455 %}
22456
22457 //------------------------------------- VectorTest --------------------------------------------
22458
22459 instruct vptest_lt16(rFlagsRegU cr, legVec src1, legVec src2, legVec vtmp) %{
22460 predicate(Matcher::vector_length_in_bytes(n->in(1)) < 16);
22461 match(Set cr (VectorTest src1 src2));
22462 effect(TEMP vtmp);
22463 format %{ "vptest_lt16 $src1, $src2\t! using $vtmp as TEMP" %}
22464 ins_encode %{
22465 BasicType bt = Matcher::vector_element_basic_type(this, $src1);
22466 int vlen = Matcher::vector_length_in_bytes(this, $src1);
22467 __ vectortest(bt, $src1$$XMMRegister, $src2$$XMMRegister, $vtmp$$XMMRegister, vlen);
22468 %}
22469 ins_pipe( pipe_slow );
22470 %}
22471
22472 instruct vptest_ge16(rFlagsRegU cr, legVec src1, legVec src2) %{
22473 predicate(Matcher::vector_length_in_bytes(n->in(1)) >= 16);
22474 match(Set cr (VectorTest src1 src2));
22475 format %{ "vptest_ge16 $src1, $src2\n\t" %}
22476 ins_encode %{
22477 BasicType bt = Matcher::vector_element_basic_type(this, $src1);
22478 int vlen = Matcher::vector_length_in_bytes(this, $src1);
22479 __ vectortest(bt, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, vlen);
22480 %}
22481 ins_pipe( pipe_slow );
22482 %}
22483
22484 instruct ktest_alltrue_le8(rFlagsRegU cr, kReg src1, kReg src2, rRegI tmp) %{
22485 predicate((Matcher::vector_length(n->in(1)) < 8 ||
22486 (Matcher::vector_length(n->in(1)) == 8 && !VM_Version::supports_avx512dq())) &&
22487 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
22488 match(Set cr (VectorTest src1 src2));
22489 effect(TEMP tmp);
22490 format %{ "ktest_alltrue_le8 $src1, $src2\t! using $tmp as TEMP" %}
22491 ins_encode %{
22492 uint masklen = Matcher::vector_length(this, $src1);
22493 __ kmovwl($tmp$$Register, $src1$$KRegister);
22494 __ andl($tmp$$Register, (1 << masklen) - 1);
22495 __ cmpl($tmp$$Register, (1 << masklen) - 1);
22496 %}
22497 ins_pipe( pipe_slow );
22498 %}
22499
22500 instruct ktest_anytrue_le8(rFlagsRegU cr, kReg src1, kReg src2, rRegI tmp) %{
22501 predicate((Matcher::vector_length(n->in(1)) < 8 ||
22502 (Matcher::vector_length(n->in(1)) == 8 && !VM_Version::supports_avx512dq())) &&
22503 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
22504 match(Set cr (VectorTest src1 src2));
22505 effect(TEMP tmp);
22506 format %{ "ktest_anytrue_le8 $src1, $src2\t! using $tmp as TEMP" %}
22507 ins_encode %{
22508 uint masklen = Matcher::vector_length(this, $src1);
22509 __ kmovwl($tmp$$Register, $src1$$KRegister);
22510 __ andl($tmp$$Register, (1 << masklen) - 1);
22511 %}
22512 ins_pipe( pipe_slow );
22513 %}
22514
22515 instruct ktest_ge8(rFlagsRegU cr, kReg src1, kReg src2) %{
22516 predicate(Matcher::vector_length(n->in(1)) >= 16 ||
22517 (Matcher::vector_length(n->in(1)) == 8 && VM_Version::supports_avx512dq()));
22518 match(Set cr (VectorTest src1 src2));
22519 format %{ "ktest_ge8 $src1, $src2\n\t" %}
22520 ins_encode %{
22521 uint masklen = Matcher::vector_length(this, $src1);
22522 __ kortest(masklen, $src1$$KRegister, $src1$$KRegister);
22523 %}
22524 ins_pipe( pipe_slow );
22525 %}
22526
22527 //------------------------------------- LoadMask --------------------------------------------
22528
22529 instruct loadMask(legVec dst, legVec src) %{
22530 predicate(n->bottom_type()->isa_pvectmask() == nullptr && !VM_Version::supports_avx512vlbw());
22531 match(Set dst (VectorLoadMask src));
22532 effect(TEMP dst);
22533 format %{ "vector_loadmask_byte $dst, $src\n\t" %}
22534 ins_encode %{
22535 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
22536 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22537 __ load_vector_mask($dst$$XMMRegister, $src$$XMMRegister, vlen_in_bytes, elem_bt, true);
22538 %}
22539 ins_pipe( pipe_slow );
22540 %}
22541
22542 instruct loadMask64(kReg dst, vec src, vec xtmp) %{
22543 predicate(n->bottom_type()->isa_pvectmask() && !VM_Version::supports_avx512vlbw());
22544 match(Set dst (VectorLoadMask src));
22545 effect(TEMP xtmp);
22546 format %{ "vector_loadmask_64byte $dst, $src\t! using $xtmp as TEMP" %}
22547 ins_encode %{
22548 __ load_vector_mask($dst$$KRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
22549 true, Assembler::AVX_512bit);
22550 %}
22551 ins_pipe( pipe_slow );
22552 %}
22553
22554 instruct loadMask_evex(kReg dst, vec src, vec xtmp) %{
22555 predicate(n->bottom_type()->isa_pvectmask() && VM_Version::supports_avx512vlbw());
22556 match(Set dst (VectorLoadMask src));
22557 effect(TEMP xtmp);
22558 format %{ "vector_loadmask_byte $dst, $src\t! using $xtmp as TEMP" %}
22559 ins_encode %{
22560 int vlen_enc = vector_length_encoding(in(1));
22561 __ load_vector_mask($dst$$KRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
22562 false, vlen_enc);
22563 %}
22564 ins_pipe( pipe_slow );
22565 %}
22566
22567 //------------------------------------- StoreMask --------------------------------------------
22568
22569 instruct vstoreMask1B(vec dst, vec src, immI_1 size) %{
22570 predicate(Matcher::vector_length(n) < 64 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22571 match(Set dst (VectorStoreMask src size));
22572 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22573 ins_encode %{
22574 int vlen = Matcher::vector_length(this);
22575 if (vlen <= 16 && UseAVX <= 2) {
22576 assert(UseSSE >= 3, "required");
22577 __ pabsb($dst$$XMMRegister, $src$$XMMRegister);
22578 } else {
22579 assert(UseAVX > 0, "required");
22580 int src_vlen_enc = vector_length_encoding(this, $src);
22581 __ vpabsb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
22582 }
22583 %}
22584 ins_pipe( pipe_slow );
22585 %}
22586
22587 instruct vstoreMask2B(vec dst, vec src, vec xtmp, immI_2 size) %{
22588 predicate(Matcher::vector_length(n) <= 16 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22589 match(Set dst (VectorStoreMask src size));
22590 effect(TEMP_DEF dst, TEMP xtmp);
22591 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22592 ins_encode %{
22593 int vlen_enc = Assembler::AVX_128bit;
22594 int vlen = Matcher::vector_length(this);
22595 if (vlen <= 8) {
22596 assert(UseSSE >= 3, "required");
22597 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
22598 __ pabsw($dst$$XMMRegister, $src$$XMMRegister);
22599 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
22600 } else {
22601 assert(UseAVX > 0, "required");
22602 __ vextracti128($dst$$XMMRegister, $src$$XMMRegister, 0x1);
22603 __ vpacksswb($dst$$XMMRegister, $src$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22604 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22605 }
22606 %}
22607 ins_pipe( pipe_slow );
22608 %}
22609
22610 instruct vstoreMask4B(vec dst, vec src, vec xtmp, immI_4 size) %{
22611 predicate(UseAVX <= 2 && Matcher::vector_length(n) <= 8 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22612 match(Set dst (VectorStoreMask src size));
22613 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22614 effect(TEMP_DEF dst, TEMP xtmp);
22615 ins_encode %{
22616 int vlen_enc = Assembler::AVX_128bit;
22617 int vlen = Matcher::vector_length(this);
22618 if (vlen <= 4) {
22619 assert(UseSSE >= 3, "required");
22620 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
22621 __ pabsd($dst$$XMMRegister, $src$$XMMRegister);
22622 __ packusdw($dst$$XMMRegister, $xtmp$$XMMRegister);
22623 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
22624 } else {
22625 assert(UseAVX > 0, "required");
22626 __ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
22627 __ vextracti128($dst$$XMMRegister, $src$$XMMRegister, 0x1);
22628 __ vpackssdw($dst$$XMMRegister, $src$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22629 __ vpacksswb($dst$$XMMRegister, $dst$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
22630 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22631 }
22632 %}
22633 ins_pipe( pipe_slow );
22634 %}
22635
22636 instruct storeMask8B(vec dst, vec src, vec xtmp, immI_8 size) %{
22637 predicate(UseAVX <= 2 && Matcher::vector_length(n) == 2);
22638 match(Set dst (VectorStoreMask src size));
22639 effect(TEMP_DEF dst, TEMP xtmp);
22640 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22641 ins_encode %{
22642 assert(UseSSE >= 3, "required");
22643 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
22644 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x8);
22645 __ pabsd($dst$$XMMRegister, $dst$$XMMRegister);
22646 __ packusdw($dst$$XMMRegister, $xtmp$$XMMRegister);
22647 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
22648 %}
22649 ins_pipe( pipe_slow );
22650 %}
22651
22652 instruct storeMask8B_avx(vec dst, vec src, immI_8 size, vec vtmp) %{
22653 predicate(UseAVX <= 2 && Matcher::vector_length(n) == 4);
22654 match(Set dst (VectorStoreMask src size));
22655 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s], using $vtmp as TEMP" %}
22656 effect(TEMP_DEF dst, TEMP vtmp);
22657 ins_encode %{
22658 int vlen_enc = Assembler::AVX_128bit;
22659 __ vshufps($dst$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 0x88, Assembler::AVX_256bit);
22660 __ vextracti128($vtmp$$XMMRegister, $dst$$XMMRegister, 0x1);
22661 __ vblendps($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0xC, vlen_enc);
22662 __ vpxor($vtmp$$XMMRegister, $vtmp$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22663 __ vpackssdw($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22664 __ vpacksswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22665 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22666 %}
22667 ins_pipe( pipe_slow );
22668 %}
22669
22670 instruct vstoreMask4B_evex_novectmask(vec dst, vec src, immI_4 size) %{
22671 predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22672 match(Set dst (VectorStoreMask src size));
22673 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22674 ins_encode %{
22675 int src_vlen_enc = vector_length_encoding(this, $src);
22676 int dst_vlen_enc = vector_length_encoding(this);
22677 if (!VM_Version::supports_avx512vl()) {
22678 src_vlen_enc = Assembler::AVX_512bit;
22679 }
22680 __ evpmovdb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
22681 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
22682 %}
22683 ins_pipe( pipe_slow );
22684 %}
22685
22686 instruct vstoreMask8B_evex_novectmask(vec dst, vec src, immI_8 size) %{
22687 predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22688 match(Set dst (VectorStoreMask src size));
22689 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22690 ins_encode %{
22691 int src_vlen_enc = vector_length_encoding(this, $src);
22692 int dst_vlen_enc = vector_length_encoding(this);
22693 if (!VM_Version::supports_avx512vl()) {
22694 src_vlen_enc = Assembler::AVX_512bit;
22695 }
22696 __ evpmovqb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
22697 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
22698 %}
22699 ins_pipe( pipe_slow );
22700 %}
22701
22702 instruct vstoreMask_evex_vectmask(vec dst, kReg mask, immI size) %{
22703 predicate(n->in(1)->bottom_type()->isa_pvectmask() && !VM_Version::supports_avx512vlbw());
22704 match(Set dst (VectorStoreMask mask size));
22705 effect(TEMP_DEF dst);
22706 format %{ "vector_store_mask $dst, $mask \t! elem size is $size byte[s]" %}
22707 ins_encode %{
22708 assert(Matcher::vector_length_in_bytes(this, $mask) == 64, "");
22709 __ evmovdqul($dst$$XMMRegister, $mask$$KRegister, ExternalAddress(vector_int_mask_cmp_bits()),
22710 false, Assembler::AVX_512bit, noreg);
22711 __ evpmovdb($dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_512bit);
22712 %}
22713 ins_pipe( pipe_slow );
22714 %}
22715
22716 instruct vstoreMask_evex(vec dst, kReg mask, immI size) %{
22717 predicate(n->in(1)->bottom_type()->isa_pvectmask() && VM_Version::supports_avx512vlbw());
22718 match(Set dst (VectorStoreMask mask size));
22719 effect(TEMP_DEF dst);
22720 format %{ "vector_store_mask $dst, $mask \t! elem size is $size byte[s]" %}
22721 ins_encode %{
22722 int dst_vlen_enc = vector_length_encoding(this);
22723 __ evpmovm2b($dst$$XMMRegister, $mask$$KRegister, dst_vlen_enc);
22724 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
22725 %}
22726 ins_pipe( pipe_slow );
22727 %}
22728
22729 instruct vmaskcast_evex(kReg dst) %{
22730 match(Set dst (VectorMaskCast dst));
22731 ins_cost(0);
22732 format %{ "vector_mask_cast $dst" %}
22733 ins_encode %{
22734 // empty
22735 %}
22736 ins_pipe(empty);
22737 %}
22738
22739 instruct vmaskcast(vec dst) %{
22740 predicate(Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
22741 match(Set dst (VectorMaskCast dst));
22742 ins_cost(0);
22743 format %{ "vector_mask_cast $dst" %}
22744 ins_encode %{
22745 // empty
22746 %}
22747 ins_pipe(empty);
22748 %}
22749
22750 instruct vmaskcast_avx(vec dst, vec src) %{
22751 predicate(Matcher::vector_length_in_bytes(n) != Matcher::vector_length_in_bytes(n->in(1)));
22752 match(Set dst (VectorMaskCast src));
22753 format %{ "vector_mask_cast $dst, $src" %}
22754 ins_encode %{
22755 int vlen = Matcher::vector_length(this);
22756 BasicType src_bt = Matcher::vector_element_basic_type(this, $src);
22757 BasicType dst_bt = Matcher::vector_element_basic_type(this);
22758 __ vector_mask_cast($dst$$XMMRegister, $src$$XMMRegister, dst_bt, src_bt, vlen);
22759 %}
22760 ins_pipe(pipe_slow);
22761 %}
22762
22763 //-------------------------------- Load Iota Indices ----------------------------------
22764
22765 instruct loadIotaIndices(vec dst, immI_0 src) %{
22766 match(Set dst (VectorLoadConst src));
22767 format %{ "vector_load_iota $dst CONSTANT_MEMORY\t! load iota indices" %}
22768 ins_encode %{
22769 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
22770 BasicType bt = Matcher::vector_element_basic_type(this);
22771 __ load_iota_indices($dst$$XMMRegister, vlen_in_bytes, bt);
22772 %}
22773 ins_pipe( pipe_slow );
22774 %}
22775
22776 instruct VectorPopulateIndex(vec dst, rRegI src1, immI_1 src2, vec vtmp) %{
22777 match(Set dst (PopulateIndex src1 src2));
22778 effect(TEMP dst, TEMP vtmp);
22779 format %{ "vector_populate_index $dst $src1 $src2\t! using $vtmp as TEMP" %}
22780 ins_encode %{
22781 assert($src2$$constant == 1, "required");
22782 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
22783 int vlen_enc = vector_length_encoding(this);
22784 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22785 __ vpbroadcast(elem_bt, $vtmp$$XMMRegister, $src1$$Register, vlen_enc);
22786 __ load_iota_indices($dst$$XMMRegister, vlen_in_bytes, elem_bt);
22787 __ vpadd(elem_bt, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22788 %}
22789 ins_pipe( pipe_slow );
22790 %}
22791
22792 instruct VectorPopulateLIndex(vec dst, rRegL src1, immI_1 src2, vec vtmp) %{
22793 match(Set dst (PopulateIndex src1 src2));
22794 effect(TEMP dst, TEMP vtmp);
22795 format %{ "vector_populate_index $dst $src1 $src2\t! using $vtmp as TEMP" %}
22796 ins_encode %{
22797 assert($src2$$constant == 1, "required");
22798 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
22799 int vlen_enc = vector_length_encoding(this);
22800 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22801 __ vpbroadcast(elem_bt, $vtmp$$XMMRegister, $src1$$Register, vlen_enc);
22802 __ load_iota_indices($dst$$XMMRegister, vlen_in_bytes, elem_bt);
22803 __ vpadd(elem_bt, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22804 %}
22805 ins_pipe( pipe_slow );
22806 %}
22807
22808 //-------------------------------- Rearrange ----------------------------------
22809
22810 // LoadShuffle/Rearrange for Byte
22811 instruct rearrangeB(vec dst, vec shuffle) %{
22812 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
22813 Matcher::vector_length(n) < 32);
22814 match(Set dst (VectorRearrange dst shuffle));
22815 format %{ "vector_rearrange $dst, $shuffle, $dst" %}
22816 ins_encode %{
22817 assert(UseSSE >= 4, "required");
22818 __ pshufb($dst$$XMMRegister, $shuffle$$XMMRegister);
22819 %}
22820 ins_pipe( pipe_slow );
22821 %}
22822
22823 instruct rearrangeB_avx(legVec dst, legVec src, vec shuffle, legVec vtmp1, legVec vtmp2) %{
22824 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
22825 Matcher::vector_length(n) == 32 && !VM_Version::supports_avx512_vbmi());
22826 match(Set dst (VectorRearrange src shuffle));
22827 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
22828 format %{ "vector_rearrange $dst, $shuffle, $src\t! using $vtmp1, $vtmp2 as TEMP" %}
22829 ins_encode %{
22830 assert(UseAVX >= 2, "required");
22831 // Swap src into vtmp1
22832 __ vperm2i128($vtmp1$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 1);
22833 // Shuffle swapped src to get entries from other 128 bit lane
22834 __ vpshufb($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
22835 // Shuffle original src to get entries from self 128 bit lane
22836 __ vpshufb($dst$$XMMRegister, $src$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
22837 // Create a blend mask by setting high bits for entries coming from other lane in shuffle
22838 __ vpaddb($vtmp2$$XMMRegister, $shuffle$$XMMRegister, ExternalAddress(vector_byte_shufflemask()), Assembler::AVX_256bit, noreg);
22839 // Perform the blend
22840 __ vpblendvb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, Assembler::AVX_256bit);
22841 %}
22842 ins_pipe( pipe_slow );
22843 %}
22844
22845
22846 instruct rearrangeB_evex(vec dst, vec src, vec shuffle, vec xtmp1, vec xtmp2, vec xtmp3, kReg ktmp, rRegI rtmp) %{
22847 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
22848 Matcher::vector_length(n) > 32 && !VM_Version::supports_avx512_vbmi());
22849 match(Set dst (VectorRearrange src shuffle));
22850 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP ktmp, TEMP rtmp);
22851 format %{ "vector_rearrange $dst, $shuffle, $src!\t using $xtmp1, $xtmp2, $xtmp3, $rtmp and $ktmp as TEMP" %}
22852 ins_encode %{
22853 int vlen_enc = vector_length_encoding(this);
22854 __ rearrange_bytes($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister,
22855 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $xtmp3$$XMMRegister,
22856 $rtmp$$Register, $ktmp$$KRegister, vlen_enc);
22857 %}
22858 ins_pipe( pipe_slow );
22859 %}
22860
22861 instruct rearrangeB_evex_vbmi(vec dst, vec src, vec shuffle) %{
22862 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
22863 Matcher::vector_length(n) >= 32 && VM_Version::supports_avx512_vbmi());
22864 match(Set dst (VectorRearrange src shuffle));
22865 format %{ "vector_rearrange $dst, $shuffle, $src" %}
22866 ins_encode %{
22867 int vlen_enc = vector_length_encoding(this);
22868 __ vpermb($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
22869 %}
22870 ins_pipe( pipe_slow );
22871 %}
22872
22873 // LoadShuffle/Rearrange for Short
22874
22875 instruct loadShuffleS(vec dst, vec src, vec vtmp) %{
22876 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
22877 !VM_Version::supports_avx512bw());
22878 match(Set dst (VectorLoadShuffle src));
22879 effect(TEMP dst, TEMP vtmp);
22880 format %{ "vector_load_shuffle $dst, $src\t! using $vtmp as TEMP" %}
22881 ins_encode %{
22882 // Create a byte shuffle mask from short shuffle mask
22883 // only byte shuffle instruction available on these platforms
22884 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
22885 if (UseAVX == 0) {
22886 assert(vlen_in_bytes <= 16, "required");
22887 // Multiply each shuffle by two to get byte index
22888 __ movdqu($vtmp$$XMMRegister, $src$$XMMRegister);
22889 __ psllw($vtmp$$XMMRegister, 1);
22890
22891 // Duplicate to create 2 copies of byte index
22892 __ movdqu($dst$$XMMRegister, $vtmp$$XMMRegister);
22893 __ psllw($dst$$XMMRegister, 8);
22894 __ por($dst$$XMMRegister, $vtmp$$XMMRegister);
22895
22896 // Add one to get alternate byte index
22897 __ movdqu($vtmp$$XMMRegister, ExternalAddress(vector_short_shufflemask()), noreg);
22898 __ paddb($dst$$XMMRegister, $vtmp$$XMMRegister);
22899 } else {
22900 assert(UseAVX > 1 || vlen_in_bytes <= 16, "required");
22901 int vlen_enc = vector_length_encoding(this);
22902 // Multiply each shuffle by two to get byte index
22903 __ vpsllw($vtmp$$XMMRegister, $src$$XMMRegister, 1, vlen_enc);
22904
22905 // Duplicate to create 2 copies of byte index
22906 __ vpsllw($dst$$XMMRegister, $vtmp$$XMMRegister, 8, vlen_enc);
22907 __ vpor($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22908
22909 // Add one to get alternate byte index
22910 __ vpaddb($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_short_shufflemask()), vlen_enc, noreg);
22911 }
22912 %}
22913 ins_pipe( pipe_slow );
22914 %}
22915
22916 instruct rearrangeS(vec dst, vec shuffle) %{
22917 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
22918 Matcher::vector_length(n) <= 8 && !VM_Version::supports_avx512bw());
22919 match(Set dst (VectorRearrange dst shuffle));
22920 format %{ "vector_rearrange $dst, $shuffle, $dst" %}
22921 ins_encode %{
22922 assert(UseSSE >= 4, "required");
22923 __ pshufb($dst$$XMMRegister, $shuffle$$XMMRegister);
22924 %}
22925 ins_pipe( pipe_slow );
22926 %}
22927
22928 instruct rearrangeS_avx(legVec dst, legVec src, vec shuffle, legVec vtmp1, legVec vtmp2) %{
22929 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
22930 Matcher::vector_length(n) == 16 && !VM_Version::supports_avx512bw());
22931 match(Set dst (VectorRearrange src shuffle));
22932 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
22933 format %{ "vector_rearrange $dst, $shuffle, $src\t! using $vtmp1, $vtmp2 as TEMP" %}
22934 ins_encode %{
22935 assert(UseAVX >= 2, "required");
22936 // Swap src into vtmp1
22937 __ vperm2i128($vtmp1$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 1);
22938 // Shuffle swapped src to get entries from other 128 bit lane
22939 __ vpshufb($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
22940 // Shuffle original src to get entries from self 128 bit lane
22941 __ vpshufb($dst$$XMMRegister, $src$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
22942 // Create a blend mask by setting high bits for entries coming from other lane in shuffle
22943 __ vpaddb($vtmp2$$XMMRegister, $shuffle$$XMMRegister, ExternalAddress(vector_byte_shufflemask()), Assembler::AVX_256bit, noreg);
22944 // Perform the blend
22945 __ vpblendvb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, Assembler::AVX_256bit);
22946 %}
22947 ins_pipe( pipe_slow );
22948 %}
22949
22950 instruct rearrangeS_evex(vec dst, vec src, vec shuffle) %{
22951 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
22952 VM_Version::supports_avx512bw());
22953 match(Set dst (VectorRearrange src shuffle));
22954 format %{ "vector_rearrange $dst, $shuffle, $src" %}
22955 ins_encode %{
22956 int vlen_enc = vector_length_encoding(this);
22957 if (!VM_Version::supports_avx512vl()) {
22958 vlen_enc = Assembler::AVX_512bit;
22959 }
22960 __ vpermw($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
22961 %}
22962 ins_pipe( pipe_slow );
22963 %}
22964
22965 // LoadShuffle/Rearrange for Integer and Float
22966
22967 instruct loadShuffleI(vec dst, vec src, vec vtmp) %{
22968 predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
22969 Matcher::vector_length(n) == 4 && UseAVX == 0);
22970 match(Set dst (VectorLoadShuffle src));
22971 effect(TEMP dst, TEMP vtmp);
22972 format %{ "vector_load_shuffle $dst, $src\t! using $vtmp as TEMP" %}
22973 ins_encode %{
22974 assert(UseSSE >= 4, "required");
22975
22976 // Create a byte shuffle mask from int shuffle mask
22977 // only byte shuffle instruction available on these platforms
22978
22979 // Duplicate and multiply each shuffle by 4
22980 __ movdqu($vtmp$$XMMRegister, $src$$XMMRegister);
22981 __ pshuflw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 0xA0);
22982 __ pshufhw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 0xA0);
22983 __ psllw($vtmp$$XMMRegister, 2);
22984
22985 // Duplicate again to create 4 copies of byte index
22986 __ movdqu($dst$$XMMRegister, $vtmp$$XMMRegister);
22987 __ psllw($dst$$XMMRegister, 8);
22988 __ por($vtmp$$XMMRegister, $dst$$XMMRegister);
22989
22990 // Add 3,2,1,0 to get alternate byte index
22991 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_int_shufflemask()), noreg);
22992 __ paddb($dst$$XMMRegister, $vtmp$$XMMRegister);
22993 %}
22994 ins_pipe( pipe_slow );
22995 %}
22996
22997 instruct rearrangeI(vec dst, vec shuffle) %{
22998 predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
22999 UseAVX == 0);
23000 match(Set dst (VectorRearrange dst shuffle));
23001 format %{ "vector_rearrange $dst, $shuffle, $dst" %}
23002 ins_encode %{
23003 assert(UseSSE >= 4, "required");
23004 __ pshufb($dst$$XMMRegister, $shuffle$$XMMRegister);
23005 %}
23006 ins_pipe( pipe_slow );
23007 %}
23008
23009 instruct rearrangeI_avx(vec dst, vec src, vec shuffle) %{
23010 predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
23011 UseAVX > 0);
23012 match(Set dst (VectorRearrange src shuffle));
23013 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23014 ins_encode %{
23015 int vlen_enc = vector_length_encoding(this);
23016 BasicType bt = Matcher::vector_element_basic_type(this);
23017 __ vector_rearrange_int_float(bt, $dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23018 %}
23019 ins_pipe( pipe_slow );
23020 %}
23021
23022 // LoadShuffle/Rearrange for Long and Double
23023
23024 instruct loadShuffleL(vec dst, vec src, vec vtmp) %{
23025 predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
23026 Matcher::vector_length(n) < 8 && !VM_Version::supports_avx512vl());
23027 match(Set dst (VectorLoadShuffle src));
23028 effect(TEMP dst, TEMP vtmp);
23029 format %{ "vector_load_shuffle $dst, $src\t! using $vtmp as TEMP" %}
23030 ins_encode %{
23031 assert(UseAVX >= 2, "required");
23032
23033 int vlen_enc = vector_length_encoding(this);
23034 // Create a double word shuffle mask from long shuffle mask
23035 // only double word shuffle instruction available on these platforms
23036
23037 // Multiply each shuffle by two to get double word index
23038 __ vpsllq($vtmp$$XMMRegister, $src$$XMMRegister, 1, vlen_enc);
23039
23040 // Duplicate each double word shuffle
23041 __ vpsllq($dst$$XMMRegister, $vtmp$$XMMRegister, 32, vlen_enc);
23042 __ vpor($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
23043
23044 // Add one to get alternate double word index
23045 __ vpaddd($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_long_shufflemask()), vlen_enc, noreg);
23046 %}
23047 ins_pipe( pipe_slow );
23048 %}
23049
23050 instruct rearrangeL(vec dst, vec src, vec shuffle) %{
23051 predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
23052 Matcher::vector_length(n) < 8 && !VM_Version::supports_avx512vl());
23053 match(Set dst (VectorRearrange src shuffle));
23054 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23055 ins_encode %{
23056 assert(UseAVX >= 2, "required");
23057
23058 int vlen_enc = vector_length_encoding(this);
23059 __ vpermd($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23060 %}
23061 ins_pipe( pipe_slow );
23062 %}
23063
23064 instruct rearrangeL_evex(vec dst, vec src, vec shuffle) %{
23065 predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
23066 (Matcher::vector_length(n) == 8 || VM_Version::supports_avx512vl()));
23067 match(Set dst (VectorRearrange src shuffle));
23068 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23069 ins_encode %{
23070 assert(UseAVX > 2, "required");
23071
23072 int vlen_enc = vector_length_encoding(this);
23073 if (vlen_enc == Assembler::AVX_128bit) {
23074 vlen_enc = Assembler::AVX_256bit;
23075 }
23076 __ vpermq($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23077 %}
23078 ins_pipe( pipe_slow );
23079 %}
23080
23081 // --------------------------------- FMA --------------------------------------
23082 // a * b + c
23083
23084 instruct vfmaF_reg(vec a, vec b, vec c) %{
23085 match(Set c (FmaVF c (Binary a b)));
23086 format %{ "fmaps $a,$b,$c\t# $c = $a * $b + $c fma packedF" %}
23087 ins_cost(150);
23088 ins_encode %{
23089 assert(UseFMA, "not enabled");
23090 int vlen_enc = vector_length_encoding(this);
23091 __ vfmaf($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister, vlen_enc);
23092 %}
23093 ins_pipe( pipe_slow );
23094 %}
23095
23096 instruct vfmaF_mem(vec a, memory b, vec c) %{
23097 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
23098 match(Set c (FmaVF c (Binary a (LoadVector b))));
23099 format %{ "fmaps $a,$b,$c\t# $c = $a * $b + $c fma packedF" %}
23100 ins_cost(150);
23101 ins_encode %{
23102 assert(UseFMA, "not enabled");
23103 int vlen_enc = vector_length_encoding(this);
23104 __ vfmaf($c$$XMMRegister, $a$$XMMRegister, $b$$Address, $c$$XMMRegister, vlen_enc);
23105 %}
23106 ins_pipe( pipe_slow );
23107 %}
23108
23109 instruct vfmaD_reg(vec a, vec b, vec c) %{
23110 match(Set c (FmaVD c (Binary a b)));
23111 format %{ "fmapd $a,$b,$c\t# $c = $a * $b + $c fma packedD" %}
23112 ins_cost(150);
23113 ins_encode %{
23114 assert(UseFMA, "not enabled");
23115 int vlen_enc = vector_length_encoding(this);
23116 __ vfmad($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister, vlen_enc);
23117 %}
23118 ins_pipe( pipe_slow );
23119 %}
23120
23121 instruct vfmaD_mem(vec a, memory b, vec c) %{
23122 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
23123 match(Set c (FmaVD c (Binary a (LoadVector b))));
23124 format %{ "fmapd $a,$b,$c\t# $c = $a * $b + $c fma packedD" %}
23125 ins_cost(150);
23126 ins_encode %{
23127 assert(UseFMA, "not enabled");
23128 int vlen_enc = vector_length_encoding(this);
23129 __ vfmad($c$$XMMRegister, $a$$XMMRegister, $b$$Address, $c$$XMMRegister, vlen_enc);
23130 %}
23131 ins_pipe( pipe_slow );
23132 %}
23133
23134 // --------------------------------- Vector Multiply Add --------------------------------------
23135
23136 instruct vmuladdS2I_reg_sse(vec dst, vec src1) %{
23137 predicate(UseAVX == 0);
23138 match(Set dst (MulAddVS2VI dst src1));
23139 format %{ "pmaddwd $dst,$src1\t! muladd packedStoI" %}
23140 ins_encode %{
23141 __ pmaddwd($dst$$XMMRegister, $src1$$XMMRegister);
23142 %}
23143 ins_pipe( pipe_slow );
23144 %}
23145
23146 instruct vmuladdS2I_reg_avx(vec dst, vec src1, vec src2) %{
23147 predicate(UseAVX > 0);
23148 match(Set dst (MulAddVS2VI src1 src2));
23149 format %{ "vpmaddwd $dst,$src1,$src2\t! muladd packedStoI" %}
23150 ins_encode %{
23151 int vlen_enc = vector_length_encoding(this);
23152 __ vpmaddwd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
23153 %}
23154 ins_pipe( pipe_slow );
23155 %}
23156
23157 // --------------------------------- Vector Multiply Add Add ----------------------------------
23158
23159 instruct vmuladdaddS2I_reg(vec dst, vec src1, vec src2) %{
23160 predicate(VM_Version::supports_avx512_vnni());
23161 match(Set dst (AddVI (MulAddVS2VI src1 src2) dst));
23162 format %{ "evpdpwssd $dst,$src1,$src2\t! muladdadd packedStoI" %}
23163 ins_encode %{
23164 assert(UseAVX > 2, "required");
23165 int vlen_enc = vector_length_encoding(this);
23166 __ evpdpwssd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
23167 %}
23168 ins_pipe( pipe_slow );
23169 ins_cost(10);
23170 %}
23171
23172 // --------------------------------- PopCount --------------------------------------
23173
23174 instruct vpopcount_integral_reg_evex(vec dst, vec src) %{
23175 predicate(is_vector_popcount_predicate(Matcher::vector_element_basic_type(n->in(1))));
23176 match(Set dst (PopCountVI src));
23177 match(Set dst (PopCountVL src));
23178 format %{ "vector_popcount_integral $dst, $src" %}
23179 ins_encode %{
23180 int opcode = this->ideal_Opcode();
23181 int vlen_enc = vector_length_encoding(this, $src);
23182 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23183 __ vector_popcount_integral_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, k0, true, vlen_enc);
23184 %}
23185 ins_pipe( pipe_slow );
23186 %}
23187
23188 instruct vpopcount_integral_reg_evex_masked(vec dst, vec src, kReg mask) %{
23189 predicate(is_vector_popcount_predicate(Matcher::vector_element_basic_type(n->in(1))));
23190 match(Set dst (PopCountVI src mask));
23191 match(Set dst (PopCountVL src mask));
23192 format %{ "vector_popcount_integral_masked $dst, $src, $mask" %}
23193 ins_encode %{
23194 int vlen_enc = vector_length_encoding(this, $src);
23195 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23196 __ evmovdquq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
23197 __ vector_popcount_integral_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $mask$$KRegister, true, vlen_enc);
23198 %}
23199 ins_pipe( pipe_slow );
23200 %}
23201
23202 instruct vpopcount_avx_reg(vec dst, vec src, vec xtmp1, vec xtmp2, rRegP rtmp) %{
23203 predicate(!is_vector_popcount_predicate(Matcher::vector_element_basic_type(n->in(1))));
23204 match(Set dst (PopCountVI src));
23205 match(Set dst (PopCountVL src));
23206 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp);
23207 format %{ "vector_popcount_integral $dst, $src\t! using $xtmp1, $xtmp2, and $rtmp as TEMP" %}
23208 ins_encode %{
23209 int opcode = this->ideal_Opcode();
23210 int vlen_enc = vector_length_encoding(this, $src);
23211 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23212 __ vector_popcount_integral(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23213 $xtmp2$$XMMRegister, $rtmp$$Register, vlen_enc);
23214 %}
23215 ins_pipe( pipe_slow );
23216 %}
23217
23218 // --------------------------------- Vector Trailing Zeros Count --------------------------------------
23219
23220 instruct vcount_trailing_zeros_reg_evex(vec dst, vec src, vec xtmp, rRegP rtmp) %{
23221 predicate(is_clz_non_subword_predicate_evex(Matcher::vector_element_basic_type(n->in(1)),
23222 Matcher::vector_length_in_bytes(n->in(1))));
23223 match(Set dst (CountTrailingZerosV src));
23224 effect(TEMP dst, TEMP xtmp, TEMP rtmp);
23225 ins_cost(400);
23226 format %{ "vector_count_trailing_zeros $dst, $src!\t using $xtmp and $rtmp as TEMP" %}
23227 ins_encode %{
23228 int vlen_enc = vector_length_encoding(this, $src);
23229 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23230 __ vector_count_trailing_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, xnoreg,
23231 xnoreg, xnoreg, $xtmp$$XMMRegister, k0, $rtmp$$Register, vlen_enc);
23232 %}
23233 ins_pipe( pipe_slow );
23234 %}
23235
23236 instruct vcount_trailing_zeros_short_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, rRegP rtmp) %{
23237 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_SHORT &&
23238 VM_Version::supports_avx512cd() &&
23239 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64));
23240 match(Set dst (CountTrailingZerosV src));
23241 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp);
23242 ins_cost(400);
23243 format %{ "vector_count_trailing_zeros $dst, $src!\t using $xtmp1, $xtmp2, $xtmp3 and $rtmp as TEMP" %}
23244 ins_encode %{
23245 int vlen_enc = vector_length_encoding(this, $src);
23246 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23247 __ vector_count_trailing_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23248 $xtmp2$$XMMRegister, xnoreg, $xtmp3$$XMMRegister, k0, $rtmp$$Register, vlen_enc);
23249 %}
23250 ins_pipe( pipe_slow );
23251 %}
23252
23253 instruct vcount_trailing_zeros_byte_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, kReg ktmp, rRegP rtmp) %{
23254 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_BYTE && VM_Version::supports_avx512vlbw());
23255 match(Set dst (CountTrailingZerosV src));
23256 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, TEMP ktmp, TEMP rtmp);
23257 ins_cost(400);
23258 format %{ "vector_count_trailing_zeros $dst, $src!\t using $xtmp1, $xtmp2, $xtmp3, $xtmp4, $ktmp and $rtmp as TEMP" %}
23259 ins_encode %{
23260 int vlen_enc = vector_length_encoding(this, $src);
23261 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23262 __ vector_count_trailing_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23263 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister,
23264 $ktmp$$KRegister, $rtmp$$Register, vlen_enc);
23265 %}
23266 ins_pipe( pipe_slow );
23267 %}
23268
23269 instruct vcount_trailing_zeros_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, rRegP rtmp) %{
23270 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n->in(1)) < 64);
23271 match(Set dst (CountTrailingZerosV src));
23272 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp);
23273 format %{ "vector_count_trailing_zeros $dst, $src\t! using $xtmp1, $xtmp2, $xtmp3, and $rtmp as TEMP" %}
23274 ins_encode %{
23275 int vlen_enc = vector_length_encoding(this, $src);
23276 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23277 __ vector_count_trailing_zeros_avx(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23278 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, vlen_enc);
23279 %}
23280 ins_pipe( pipe_slow );
23281 %}
23282
23283
23284 // --------------------------------- Bitwise Ternary Logic ----------------------------------
23285
23286 instruct vpternlog(vec dst, vec src2, vec src3, immU8 func) %{
23287 match(Set dst (MacroLogicV (Binary dst src2) (Binary src3 func)));
23288 effect(TEMP dst);
23289 format %{ "vpternlogd $dst,$src2,$src3,$func\t! vector ternary logic" %}
23290 ins_encode %{
23291 int vector_len = vector_length_encoding(this);
23292 __ vpternlogd($dst$$XMMRegister, $func$$constant, $src2$$XMMRegister, $src3$$XMMRegister, vector_len);
23293 %}
23294 ins_pipe( pipe_slow );
23295 %}
23296
23297 instruct vpternlog_mem(vec dst, vec src2, memory src3, immU8 func) %{
23298 predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) > 8);
23299 match(Set dst (MacroLogicV (Binary dst src2) (Binary (LoadVector src3) func)));
23300 effect(TEMP dst);
23301 format %{ "vpternlogd $dst,$src2,$src3,$func\t! vector ternary logic" %}
23302 ins_encode %{
23303 int vector_len = vector_length_encoding(this);
23304 __ vpternlogd($dst$$XMMRegister, $func$$constant, $src2$$XMMRegister, $src3$$Address, vector_len);
23305 %}
23306 ins_pipe( pipe_slow );
23307 %}
23308
23309 // --------------------------------- Rotation Operations ----------------------------------
23310 instruct vprotate_immI8(vec dst, vec src, immI8 shift) %{
23311 match(Set dst (RotateLeftV src shift));
23312 match(Set dst (RotateRightV src shift));
23313 format %{ "vprotate_imm8 $dst,$src,$shift\t! vector rotate" %}
23314 ins_encode %{
23315 int opcode = this->ideal_Opcode();
23316 int vector_len = vector_length_encoding(this);
23317 BasicType etype = this->bottom_type()->is_vect()->element_basic_type();
23318 __ vprotate_imm(opcode, etype, $dst$$XMMRegister, $src$$XMMRegister, $shift$$constant, vector_len);
23319 %}
23320 ins_pipe( pipe_slow );
23321 %}
23322
23323 instruct vprorate(vec dst, vec src, vec shift) %{
23324 match(Set dst (RotateLeftV src shift));
23325 match(Set dst (RotateRightV src shift));
23326 format %{ "vprotate $dst,$src,$shift\t! vector rotate" %}
23327 ins_encode %{
23328 int opcode = this->ideal_Opcode();
23329 int vector_len = vector_length_encoding(this);
23330 BasicType etype = this->bottom_type()->is_vect()->element_basic_type();
23331 __ vprotate_var(opcode, etype, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
23332 %}
23333 ins_pipe( pipe_slow );
23334 %}
23335
23336 // ---------------------------------- Masked Operations ------------------------------------
23337 instruct vmasked_load_avx_non_subword(vec dst, memory mem, vec mask) %{
23338 predicate(!n->in(3)->bottom_type()->isa_pvectmask());
23339 match(Set dst (LoadVectorMasked mem mask));
23340 format %{ "vector_masked_load $dst, $mem, $mask \t! vector masked copy" %}
23341 ins_encode %{
23342 BasicType elmType = this->bottom_type()->is_vect()->element_basic_type();
23343 int vlen_enc = vector_length_encoding(this);
23344 __ vmovmask(elmType, $dst$$XMMRegister, $mem$$Address, $mask$$XMMRegister, vlen_enc);
23345 %}
23346 ins_pipe( pipe_slow );
23347 %}
23348
23349
23350 instruct vmasked_load_evex(vec dst, memory mem, kReg mask) %{
23351 predicate(n->in(3)->bottom_type()->isa_pvectmask());
23352 match(Set dst (LoadVectorMasked mem mask));
23353 format %{ "vector_masked_load $dst, $mem, $mask \t! vector masked copy" %}
23354 ins_encode %{
23355 BasicType elmType = this->bottom_type()->is_vect()->element_basic_type();
23356 int vector_len = vector_length_encoding(this);
23357 __ evmovdqu(elmType, $mask$$KRegister, $dst$$XMMRegister, $mem$$Address, false, vector_len);
23358 %}
23359 ins_pipe( pipe_slow );
23360 %}
23361
23362 instruct vmasked_store_avx_non_subword(memory mem, vec src, vec mask) %{
23363 predicate(!n->in(3)->in(2)->bottom_type()->isa_pvectmask());
23364 match(Set mem (StoreVectorMasked mem (Binary src mask)));
23365 format %{ "vector_masked_store $mem, $src, $mask \t! vector masked store" %}
23366 ins_encode %{
23367 const MachNode* src_node = static_cast<const MachNode*>(this->in(this->operand_index($src)));
23368 int vlen_enc = vector_length_encoding(src_node);
23369 BasicType elmType = src_node->bottom_type()->is_vect()->element_basic_type();
23370 __ vmovmask(elmType, $mem$$Address, $src$$XMMRegister, $mask$$XMMRegister, vlen_enc);
23371 %}
23372 ins_pipe( pipe_slow );
23373 %}
23374
23375 instruct vmasked_store_evex(memory mem, vec src, kReg mask) %{
23376 predicate(n->in(3)->in(2)->bottom_type()->isa_pvectmask());
23377 match(Set mem (StoreVectorMasked mem (Binary src mask)));
23378 format %{ "vector_masked_store $mem, $src, $mask \t! vector masked store" %}
23379 ins_encode %{
23380 const MachNode* src_node = static_cast<const MachNode*>(this->in(this->operand_index($src)));
23381 BasicType elmType = src_node->bottom_type()->is_vect()->element_basic_type();
23382 int vlen_enc = vector_length_encoding(src_node);
23383 __ evmovdqu(elmType, $mask$$KRegister, $mem$$Address, $src$$XMMRegister, true, vlen_enc);
23384 %}
23385 ins_pipe( pipe_slow );
23386 %}
23387
23388 instruct verify_vector_alignment(rRegP addr, immL32 mask, rFlagsReg cr) %{
23389 match(Set addr (VerifyVectorAlignment addr mask));
23390 effect(KILL cr);
23391 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
23392 ins_encode %{
23393 Label Lskip;
23394 // check if masked bits of addr are zero
23395 __ testq($addr$$Register, $mask$$constant);
23396 __ jccb(Assembler::equal, Lskip);
23397 __ stop("verify_vector_alignment found a misaligned vector memory access");
23398 __ bind(Lskip);
23399 %}
23400 ins_pipe(pipe_slow);
23401 %}
23402
23403 instruct vmask_cmp_node(rRegI dst, vec src1, vec src2, kReg mask, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
23404 match(Set dst (VectorCmpMasked src1 (Binary src2 mask)));
23405 effect(TEMP_DEF dst, TEMP ktmp1, TEMP ktmp2, KILL cr);
23406 format %{ "vector_mask_cmp $src1, $src2, $mask \t! vector mask comparison" %}
23407 ins_encode %{
23408 assert(vector_length_encoding(this, $src1) == vector_length_encoding(this, $src2), "mismatch");
23409 assert(Matcher::vector_element_basic_type(this, $src1) == Matcher::vector_element_basic_type(this, $src2), "mismatch");
23410
23411 Label DONE;
23412 int vlen_enc = vector_length_encoding(this, $src1);
23413 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src1);
23414
23415 __ knotql($ktmp2$$KRegister, $mask$$KRegister);
23416 __ mov64($dst$$Register, -1L);
23417 __ evpcmp(elem_bt, $ktmp1$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, Assembler::eq, vlen_enc);
23418 __ kortestql($ktmp2$$KRegister, $ktmp1$$KRegister);
23419 __ jccb(Assembler::carrySet, DONE);
23420 __ kmovql($dst$$Register, $ktmp1$$KRegister);
23421 __ notq($dst$$Register);
23422 __ tzcntq($dst$$Register, $dst$$Register);
23423 __ bind(DONE);
23424 %}
23425 ins_pipe( pipe_slow );
23426 %}
23427
23428
23429 instruct vmask_gen(kReg dst, rRegL len, rRegL temp, rFlagsReg cr) %{
23430 match(Set dst (VectorMaskGen len));
23431 effect(TEMP temp, KILL cr);
23432 format %{ "vector_mask_gen32 $dst, $len \t! vector mask generator" %}
23433 ins_encode %{
23434 __ genmask($dst$$KRegister, $len$$Register, $temp$$Register);
23435 %}
23436 ins_pipe( pipe_slow );
23437 %}
23438
23439 instruct vmask_gen_imm(kReg dst, immL len, rRegL temp) %{
23440 match(Set dst (VectorMaskGen len));
23441 format %{ "vector_mask_gen $len \t! vector mask generator" %}
23442 effect(TEMP temp);
23443 ins_encode %{
23444 if ($len$$constant > 0) {
23445 __ mov64($temp$$Register, right_n_bits($len$$constant));
23446 __ kmovql($dst$$KRegister, $temp$$Register);
23447 } else {
23448 __ kxorql($dst$$KRegister, $dst$$KRegister, $dst$$KRegister);
23449 }
23450 %}
23451 ins_pipe( pipe_slow );
23452 %}
23453
23454 instruct vmask_tolong_evex(rRegL dst, kReg mask, rFlagsReg cr) %{
23455 predicate(n->in(1)->bottom_type()->isa_pvectmask());
23456 match(Set dst (VectorMaskToLong mask));
23457 effect(TEMP dst, KILL cr);
23458 format %{ "vector_tolong_evex $dst, $mask \t! vector mask tolong" %}
23459 ins_encode %{
23460 int opcode = this->ideal_Opcode();
23461 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23462 int mask_len = Matcher::vector_length(this, $mask);
23463 int mask_size = mask_len * type2aelembytes(mbt);
23464 int vlen_enc = vector_length_encoding(this, $mask);
23465 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
23466 $dst$$Register, mask_len, mask_size, vlen_enc);
23467 %}
23468 ins_pipe( pipe_slow );
23469 %}
23470
23471 instruct vmask_tolong_bool(rRegL dst, vec mask, vec xtmp, rFlagsReg cr) %{
23472 predicate(n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23473 match(Set dst (VectorMaskToLong mask));
23474 format %{ "vector_tolong_bool $dst, $mask \t! using $xtmp as TEMP" %}
23475 effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
23476 ins_encode %{
23477 int opcode = this->ideal_Opcode();
23478 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23479 int mask_len = Matcher::vector_length(this, $mask);
23480 int vlen_enc = vector_length_encoding(this, $mask);
23481 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23482 $dst$$Register, mask_len, mbt, vlen_enc);
23483 %}
23484 ins_pipe( pipe_slow );
23485 %}
23486
23487 instruct vmask_tolong_avx(rRegL dst, vec mask, immI size, vec xtmp, rFlagsReg cr) %{
23488 predicate(n->in(1)->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23489 match(Set dst (VectorMaskToLong (VectorStoreMask mask size)));
23490 format %{ "vector_tolong_avx $dst, $mask \t! using $xtmp as TEMP" %}
23491 effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
23492 ins_encode %{
23493 int opcode = this->ideal_Opcode();
23494 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23495 int mask_len = Matcher::vector_length(this, $mask);
23496 int vlen_enc = vector_length_encoding(this, $mask);
23497 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23498 $dst$$Register, mask_len, mbt, vlen_enc);
23499 %}
23500 ins_pipe( pipe_slow );
23501 %}
23502
23503 instruct vmask_truecount_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
23504 predicate(n->in(1)->bottom_type()->isa_pvectmask());
23505 match(Set dst (VectorMaskTrueCount mask));
23506 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
23507 format %{ "vector_truecount_evex $dst, $mask \t! using $tmp as TEMP" %}
23508 ins_encode %{
23509 int opcode = this->ideal_Opcode();
23510 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23511 int mask_len = Matcher::vector_length(this, $mask);
23512 int mask_size = mask_len * type2aelembytes(mbt);
23513 int vlen_enc = vector_length_encoding(this, $mask);
23514 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
23515 $tmp$$Register, mask_len, mask_size, vlen_enc);
23516 %}
23517 ins_pipe( pipe_slow );
23518 %}
23519
23520 instruct vmask_truecount_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23521 predicate(n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23522 match(Set dst (VectorMaskTrueCount mask));
23523 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23524 format %{ "vector_truecount_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23525 ins_encode %{
23526 int opcode = this->ideal_Opcode();
23527 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23528 int mask_len = Matcher::vector_length(this, $mask);
23529 int vlen_enc = vector_length_encoding(this, $mask);
23530 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23531 $tmp$$Register, mask_len, mbt, vlen_enc);
23532 %}
23533 ins_pipe( pipe_slow );
23534 %}
23535
23536 instruct vmask_truecount_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23537 predicate(n->in(1)->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23538 match(Set dst (VectorMaskTrueCount (VectorStoreMask mask size)));
23539 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23540 format %{ "vector_truecount_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23541 ins_encode %{
23542 int opcode = this->ideal_Opcode();
23543 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23544 int mask_len = Matcher::vector_length(this, $mask);
23545 int vlen_enc = vector_length_encoding(this, $mask);
23546 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23547 $tmp$$Register, mask_len, mbt, vlen_enc);
23548 %}
23549 ins_pipe( pipe_slow );
23550 %}
23551
23552 instruct vmask_first_or_last_true_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
23553 predicate(n->in(1)->bottom_type()->isa_pvectmask());
23554 match(Set dst (VectorMaskFirstTrue mask));
23555 match(Set dst (VectorMaskLastTrue mask));
23556 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
23557 format %{ "vector_mask_first_or_last_true_evex $dst, $mask \t! using $tmp as TEMP" %}
23558 ins_encode %{
23559 int opcode = this->ideal_Opcode();
23560 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23561 int mask_len = Matcher::vector_length(this, $mask);
23562 int mask_size = mask_len * type2aelembytes(mbt);
23563 int vlen_enc = vector_length_encoding(this, $mask);
23564 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
23565 $tmp$$Register, mask_len, mask_size, vlen_enc);
23566 %}
23567 ins_pipe( pipe_slow );
23568 %}
23569
23570 instruct vmask_first_or_last_true_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23571 predicate(n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23572 match(Set dst (VectorMaskFirstTrue mask));
23573 match(Set dst (VectorMaskLastTrue mask));
23574 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23575 format %{ "vector_mask_first_or_last_true_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23576 ins_encode %{
23577 int opcode = this->ideal_Opcode();
23578 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23579 int mask_len = Matcher::vector_length(this, $mask);
23580 int vlen_enc = vector_length_encoding(this, $mask);
23581 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23582 $tmp$$Register, mask_len, mbt, vlen_enc);
23583 %}
23584 ins_pipe( pipe_slow );
23585 %}
23586
23587 instruct vmask_first_or_last_true_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23588 predicate(n->in(1)->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23589 match(Set dst (VectorMaskFirstTrue (VectorStoreMask mask size)));
23590 match(Set dst (VectorMaskLastTrue (VectorStoreMask mask size)));
23591 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23592 format %{ "vector_mask_first_or_last_true_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23593 ins_encode %{
23594 int opcode = this->ideal_Opcode();
23595 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23596 int mask_len = Matcher::vector_length(this, $mask);
23597 int vlen_enc = vector_length_encoding(this, $mask);
23598 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23599 $tmp$$Register, mask_len, mbt, vlen_enc);
23600 %}
23601 ins_pipe( pipe_slow );
23602 %}
23603
23604 // --------------------------------- Compress/Expand Operations ---------------------------
23605 instruct vcompress_reg_avx(vec dst, vec src, vec mask, rRegI rtmp, rRegL rscratch, vec perm, vec xtmp, rFlagsReg cr) %{
23606 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n) <= 32);
23607 match(Set dst (CompressV src mask));
23608 match(Set dst (ExpandV src mask));
23609 effect(TEMP_DEF dst, TEMP perm, TEMP xtmp, TEMP rtmp, TEMP rscratch, KILL cr);
23610 format %{ "vector_compress $dst, $src, $mask \t!using $xtmp, $rtmp, $rscratch and $perm as TEMP" %}
23611 ins_encode %{
23612 int opcode = this->ideal_Opcode();
23613 int vlen_enc = vector_length_encoding(this);
23614 BasicType bt = Matcher::vector_element_basic_type(this);
23615 __ vector_compress_expand_avx2(opcode, $dst$$XMMRegister, $src$$XMMRegister, $mask$$XMMRegister, $rtmp$$Register,
23616 $rscratch$$Register, $perm$$XMMRegister, $xtmp$$XMMRegister, bt, vlen_enc);
23617 %}
23618 ins_pipe( pipe_slow );
23619 %}
23620
23621 instruct vcompress_expand_reg_evex(vec dst, vec src, kReg mask) %{
23622 predicate(VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64);
23623 match(Set dst (CompressV src mask));
23624 match(Set dst (ExpandV src mask));
23625 format %{ "vector_compress_expand $dst, $src, $mask" %}
23626 ins_encode %{
23627 int opcode = this->ideal_Opcode();
23628 int vector_len = vector_length_encoding(this);
23629 BasicType bt = Matcher::vector_element_basic_type(this);
23630 __ vector_compress_expand(opcode, $dst$$XMMRegister, $src$$XMMRegister, $mask$$KRegister, false, bt, vector_len);
23631 %}
23632 ins_pipe( pipe_slow );
23633 %}
23634
23635 instruct vcompress_mask_reg_evex(kReg dst, kReg mask, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
23636 match(Set dst (CompressM mask));
23637 effect(TEMP rtmp1, TEMP rtmp2, KILL cr);
23638 format %{ "mask_compress_evex $dst, $mask\t! using $rtmp1 and $rtmp2 as TEMP" %}
23639 ins_encode %{
23640 assert(this->in(1)->bottom_type()->isa_pvectmask(), "");
23641 int mask_len = Matcher::vector_length(this);
23642 __ vector_mask_compress($dst$$KRegister, $mask$$KRegister, $rtmp1$$Register, $rtmp2$$Register, mask_len);
23643 %}
23644 ins_pipe( pipe_slow );
23645 %}
23646
23647 // -------------------------------- Bit and Byte Reversal Vector Operations ------------------------
23648
23649 instruct vreverse_reg(vec dst, vec src, vec xtmp1, vec xtmp2, rRegI rtmp) %{
23650 predicate(!VM_Version::supports_gfni());
23651 match(Set dst (ReverseV src));
23652 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp);
23653 format %{ "vector_reverse_bit_evex $dst, $src!\t using $xtmp1, $xtmp2 and $rtmp as TEMP" %}
23654 ins_encode %{
23655 int vec_enc = vector_length_encoding(this);
23656 BasicType bt = Matcher::vector_element_basic_type(this);
23657 __ vector_reverse_bit(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23658 $xtmp2$$XMMRegister, $rtmp$$Register, vec_enc);
23659 %}
23660 ins_pipe( pipe_slow );
23661 %}
23662
23663 instruct vreverse_reg_gfni(vec dst, vec src, vec xtmp) %{
23664 predicate(VM_Version::supports_gfni());
23665 match(Set dst (ReverseV src));
23666 effect(TEMP dst, TEMP xtmp);
23667 format %{ "vector_reverse_bit_gfni $dst, $src!\t using $xtmp as TEMP" %}
23668 ins_encode %{
23669 int vec_enc = vector_length_encoding(this);
23670 BasicType bt = Matcher::vector_element_basic_type(this);
23671 InternalAddress addr = $constantaddress(jlong(0x8040201008040201));
23672 __ vector_reverse_bit_gfni(bt, $dst$$XMMRegister, $src$$XMMRegister, addr, vec_enc,
23673 $xtmp$$XMMRegister);
23674 %}
23675 ins_pipe( pipe_slow );
23676 %}
23677
23678 instruct vreverse_byte_reg(vec dst, vec src) %{
23679 predicate(VM_Version::supports_avx512bw() || Matcher::vector_length_in_bytes(n) < 64);
23680 match(Set dst (ReverseBytesV src));
23681 effect(TEMP dst);
23682 format %{ "vector_reverse_byte $dst, $src" %}
23683 ins_encode %{
23684 int vec_enc = vector_length_encoding(this);
23685 BasicType bt = Matcher::vector_element_basic_type(this);
23686 __ vector_reverse_byte(bt, $dst$$XMMRegister, $src$$XMMRegister, vec_enc);
23687 %}
23688 ins_pipe( pipe_slow );
23689 %}
23690
23691 instruct vreverse_byte64_reg(vec dst, vec src, vec xtmp1, vec xtmp2, rRegI rtmp) %{
23692 predicate(!VM_Version::supports_avx512bw() && Matcher::vector_length_in_bytes(n) == 64);
23693 match(Set dst (ReverseBytesV src));
23694 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp);
23695 format %{ "vector_reverse_byte $dst, $src!\t using $xtmp1, $xtmp2 and $rtmp as TEMP" %}
23696 ins_encode %{
23697 int vec_enc = vector_length_encoding(this);
23698 BasicType bt = Matcher::vector_element_basic_type(this);
23699 __ vector_reverse_byte64(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23700 $xtmp2$$XMMRegister, $rtmp$$Register, vec_enc);
23701 %}
23702 ins_pipe( pipe_slow );
23703 %}
23704
23705 // ---------------------------------- Vector Count Leading Zeros -----------------------------------
23706
23707 instruct vcount_leading_zeros_IL_reg_evex(vec dst, vec src) %{
23708 predicate(is_clz_non_subword_predicate_evex(Matcher::vector_element_basic_type(n->in(1)),
23709 Matcher::vector_length_in_bytes(n->in(1))));
23710 match(Set dst (CountLeadingZerosV src));
23711 format %{ "vector_count_leading_zeros $dst, $src" %}
23712 ins_encode %{
23713 int vlen_enc = vector_length_encoding(this, $src);
23714 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23715 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, xnoreg,
23716 xnoreg, xnoreg, k0, noreg, true, vlen_enc);
23717 %}
23718 ins_pipe( pipe_slow );
23719 %}
23720
23721 instruct vcount_leading_zeros_IL_reg_evex_masked(vec dst, vec src, kReg mask) %{
23722 predicate(is_clz_non_subword_predicate_evex(Matcher::vector_element_basic_type(n->in(1)),
23723 Matcher::vector_length_in_bytes(n->in(1))));
23724 match(Set dst (CountLeadingZerosV src mask));
23725 format %{ "vector_count_leading_zeros $dst, $src, $mask" %}
23726 ins_encode %{
23727 int vlen_enc = vector_length_encoding(this, $src);
23728 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23729 __ evmovdquq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
23730 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, xnoreg, xnoreg,
23731 xnoreg, $mask$$KRegister, noreg, true, vlen_enc);
23732 %}
23733 ins_pipe( pipe_slow );
23734 %}
23735
23736 instruct vcount_leading_zeros_short_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2) %{
23737 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_SHORT &&
23738 VM_Version::supports_avx512cd() &&
23739 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64));
23740 match(Set dst (CountLeadingZerosV src));
23741 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
23742 format %{ "vector_count_leading_zeros $dst, $src!\t using $xtmp1 and $xtmp2 as TEMP" %}
23743 ins_encode %{
23744 int vlen_enc = vector_length_encoding(this, $src);
23745 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23746 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23747 $xtmp2$$XMMRegister, xnoreg, k0, noreg, true, vlen_enc);
23748 %}
23749 ins_pipe( pipe_slow );
23750 %}
23751
23752 instruct vcount_leading_zeros_byte_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, kReg ktmp, rRegP rtmp) %{
23753 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_BYTE && VM_Version::supports_avx512vlbw());
23754 match(Set dst (CountLeadingZerosV src));
23755 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP ktmp, TEMP rtmp);
23756 format %{ "vector_count_leading_zeros $dst, $src!\t using $xtmp1, $xtmp2, $xtmp3, $ktmp and $rtmp as TEMP" %}
23757 ins_encode %{
23758 int vlen_enc = vector_length_encoding(this, $src);
23759 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23760 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23761 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $ktmp$$KRegister,
23762 $rtmp$$Register, true, vlen_enc);
23763 %}
23764 ins_pipe( pipe_slow );
23765 %}
23766
23767 instruct vcount_leading_zeros_int_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3) %{
23768 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_INT &&
23769 !VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n->in(1)) < 64);
23770 match(Set dst (CountLeadingZerosV src));
23771 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3);
23772 format %{ "vector_count_leading_zeros $dst, $src\t! using $xtmp1, $xtmp2 and $xtmp3 as TEMP" %}
23773 ins_encode %{
23774 int vlen_enc = vector_length_encoding(this, $src);
23775 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23776 __ vector_count_leading_zeros_avx(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23777 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, noreg, vlen_enc);
23778 %}
23779 ins_pipe( pipe_slow );
23780 %}
23781
23782 instruct vcount_leading_zeros_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, rRegP rtmp) %{
23783 predicate(Matcher::vector_element_basic_type(n->in(1)) != T_INT &&
23784 !VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n->in(1)) < 64);
23785 match(Set dst (CountLeadingZerosV src));
23786 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp);
23787 format %{ "vector_count_leading_zeros $dst, $src\t! using $xtmp1, $xtmp2, $xtmp3, and $rtmp as TEMP" %}
23788 ins_encode %{
23789 int vlen_enc = vector_length_encoding(this, $src);
23790 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23791 __ vector_count_leading_zeros_avx(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23792 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, vlen_enc);
23793 %}
23794 ins_pipe( pipe_slow );
23795 %}
23796
23797 // ---------------------------------- Vector Masked Operations ------------------------------------
23798
23799 instruct vadd_reg_masked(vec dst, vec src2, kReg mask) %{
23800 match(Set dst (AddVB (Binary dst src2) mask));
23801 match(Set dst (AddVS (Binary dst src2) mask));
23802 match(Set dst (AddVI (Binary dst src2) mask));
23803 match(Set dst (AddVL (Binary dst src2) mask));
23804 match(Set dst (AddVF (Binary dst src2) mask));
23805 match(Set dst (AddVD (Binary dst src2) mask));
23806 format %{ "vpadd_masked $dst, $dst, $src2, $mask\t! add masked operation" %}
23807 ins_encode %{
23808 int vlen_enc = vector_length_encoding(this);
23809 BasicType bt = Matcher::vector_element_basic_type(this);
23810 int opc = this->ideal_Opcode();
23811 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23812 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
23813 %}
23814 ins_pipe( pipe_slow );
23815 %}
23816
23817 instruct vadd_mem_masked(vec dst, memory src2, kReg mask) %{
23818 match(Set dst (AddVB (Binary dst (LoadVector src2)) mask));
23819 match(Set dst (AddVS (Binary dst (LoadVector src2)) mask));
23820 match(Set dst (AddVI (Binary dst (LoadVector src2)) mask));
23821 match(Set dst (AddVL (Binary dst (LoadVector src2)) mask));
23822 match(Set dst (AddVF (Binary dst (LoadVector src2)) mask));
23823 match(Set dst (AddVD (Binary dst (LoadVector src2)) mask));
23824 format %{ "vpadd_masked $dst, $dst, $src2, $mask\t! add masked operation" %}
23825 ins_encode %{
23826 int vlen_enc = vector_length_encoding(this);
23827 BasicType bt = Matcher::vector_element_basic_type(this);
23828 int opc = this->ideal_Opcode();
23829 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23830 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
23831 %}
23832 ins_pipe( pipe_slow );
23833 %}
23834
23835 instruct vxor_reg_masked(vec dst, vec src2, kReg mask) %{
23836 match(Set dst (XorV (Binary dst src2) mask));
23837 format %{ "vxor_masked $dst, $dst, $src2, $mask\t! xor masked operation" %}
23838 ins_encode %{
23839 int vlen_enc = vector_length_encoding(this);
23840 BasicType bt = Matcher::vector_element_basic_type(this);
23841 int opc = this->ideal_Opcode();
23842 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23843 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
23844 %}
23845 ins_pipe( pipe_slow );
23846 %}
23847
23848 instruct vxor_mem_masked(vec dst, memory src2, kReg mask) %{
23849 match(Set dst (XorV (Binary dst (LoadVector src2)) mask));
23850 format %{ "vxor_masked $dst, $dst, $src2, $mask\t! xor masked operation" %}
23851 ins_encode %{
23852 int vlen_enc = vector_length_encoding(this);
23853 BasicType bt = Matcher::vector_element_basic_type(this);
23854 int opc = this->ideal_Opcode();
23855 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23856 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
23857 %}
23858 ins_pipe( pipe_slow );
23859 %}
23860
23861 instruct vor_reg_masked(vec dst, vec src2, kReg mask) %{
23862 match(Set dst (OrV (Binary dst src2) mask));
23863 format %{ "vor_masked $dst, $dst, $src2, $mask\t! or masked operation" %}
23864 ins_encode %{
23865 int vlen_enc = vector_length_encoding(this);
23866 BasicType bt = Matcher::vector_element_basic_type(this);
23867 int opc = this->ideal_Opcode();
23868 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23869 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
23870 %}
23871 ins_pipe( pipe_slow );
23872 %}
23873
23874 instruct vor_mem_masked(vec dst, memory src2, kReg mask) %{
23875 match(Set dst (OrV (Binary dst (LoadVector src2)) mask));
23876 format %{ "vor_masked $dst, $dst, $src2, $mask\t! or masked operation" %}
23877 ins_encode %{
23878 int vlen_enc = vector_length_encoding(this);
23879 BasicType bt = Matcher::vector_element_basic_type(this);
23880 int opc = this->ideal_Opcode();
23881 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23882 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
23883 %}
23884 ins_pipe( pipe_slow );
23885 %}
23886
23887 instruct vand_reg_masked(vec dst, vec src2, kReg mask) %{
23888 match(Set dst (AndV (Binary dst src2) mask));
23889 format %{ "vand_masked $dst, $dst, $src2, $mask\t! and masked operation" %}
23890 ins_encode %{
23891 int vlen_enc = vector_length_encoding(this);
23892 BasicType bt = Matcher::vector_element_basic_type(this);
23893 int opc = this->ideal_Opcode();
23894 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23895 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
23896 %}
23897 ins_pipe( pipe_slow );
23898 %}
23899
23900 instruct vand_mem_masked(vec dst, memory src2, kReg mask) %{
23901 match(Set dst (AndV (Binary dst (LoadVector src2)) mask));
23902 format %{ "vand_masked $dst, $dst, $src2, $mask\t! and masked operation" %}
23903 ins_encode %{
23904 int vlen_enc = vector_length_encoding(this);
23905 BasicType bt = Matcher::vector_element_basic_type(this);
23906 int opc = this->ideal_Opcode();
23907 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23908 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
23909 %}
23910 ins_pipe( pipe_slow );
23911 %}
23912
23913 instruct vsub_reg_masked(vec dst, vec src2, kReg mask) %{
23914 match(Set dst (SubVB (Binary dst src2) mask));
23915 match(Set dst (SubVS (Binary dst src2) mask));
23916 match(Set dst (SubVI (Binary dst src2) mask));
23917 match(Set dst (SubVL (Binary dst src2) mask));
23918 match(Set dst (SubVF (Binary dst src2) mask));
23919 match(Set dst (SubVD (Binary dst src2) mask));
23920 format %{ "vpsub_masked $dst, $dst, $src2, $mask\t! sub masked operation" %}
23921 ins_encode %{
23922 int vlen_enc = vector_length_encoding(this);
23923 BasicType bt = Matcher::vector_element_basic_type(this);
23924 int opc = this->ideal_Opcode();
23925 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23926 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
23927 %}
23928 ins_pipe( pipe_slow );
23929 %}
23930
23931 instruct vsub_mem_masked(vec dst, memory src2, kReg mask) %{
23932 match(Set dst (SubVB (Binary dst (LoadVector src2)) mask));
23933 match(Set dst (SubVS (Binary dst (LoadVector src2)) mask));
23934 match(Set dst (SubVI (Binary dst (LoadVector src2)) mask));
23935 match(Set dst (SubVL (Binary dst (LoadVector src2)) mask));
23936 match(Set dst (SubVF (Binary dst (LoadVector src2)) mask));
23937 match(Set dst (SubVD (Binary dst (LoadVector src2)) mask));
23938 format %{ "vpsub_masked $dst, $dst, $src2, $mask\t! sub masked operation" %}
23939 ins_encode %{
23940 int vlen_enc = vector_length_encoding(this);
23941 BasicType bt = Matcher::vector_element_basic_type(this);
23942 int opc = this->ideal_Opcode();
23943 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23944 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
23945 %}
23946 ins_pipe( pipe_slow );
23947 %}
23948
23949 instruct vmul_reg_masked(vec dst, vec src2, kReg mask) %{
23950 match(Set dst (MulVS (Binary dst src2) mask));
23951 match(Set dst (MulVI (Binary dst src2) mask));
23952 match(Set dst (MulVL (Binary dst src2) mask));
23953 match(Set dst (MulVF (Binary dst src2) mask));
23954 match(Set dst (MulVD (Binary dst src2) mask));
23955 format %{ "vpmul_masked $dst, $dst, $src2, $mask\t! mul masked operation" %}
23956 ins_encode %{
23957 int vlen_enc = vector_length_encoding(this);
23958 BasicType bt = Matcher::vector_element_basic_type(this);
23959 int opc = this->ideal_Opcode();
23960 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23961 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
23962 %}
23963 ins_pipe( pipe_slow );
23964 %}
23965
23966 instruct vmul_mem_masked(vec dst, memory src2, kReg mask) %{
23967 match(Set dst (MulVS (Binary dst (LoadVector src2)) mask));
23968 match(Set dst (MulVI (Binary dst (LoadVector src2)) mask));
23969 match(Set dst (MulVL (Binary dst (LoadVector src2)) mask));
23970 match(Set dst (MulVF (Binary dst (LoadVector src2)) mask));
23971 match(Set dst (MulVD (Binary dst (LoadVector src2)) mask));
23972 format %{ "vpmul_masked $dst, $dst, $src2, $mask\t! mul masked operation" %}
23973 ins_encode %{
23974 int vlen_enc = vector_length_encoding(this);
23975 BasicType bt = Matcher::vector_element_basic_type(this);
23976 int opc = this->ideal_Opcode();
23977 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23978 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
23979 %}
23980 ins_pipe( pipe_slow );
23981 %}
23982
23983 instruct vsqrt_reg_masked(vec dst, kReg mask) %{
23984 match(Set dst (SqrtVF dst mask));
23985 match(Set dst (SqrtVD dst mask));
23986 format %{ "vpsqrt_masked $dst, $mask\t! sqrt masked operation" %}
23987 ins_encode %{
23988 int vlen_enc = vector_length_encoding(this);
23989 BasicType bt = Matcher::vector_element_basic_type(this);
23990 int opc = this->ideal_Opcode();
23991 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23992 $dst$$XMMRegister, $dst$$XMMRegister, true, vlen_enc);
23993 %}
23994 ins_pipe( pipe_slow );
23995 %}
23996
23997 instruct vdiv_reg_masked(vec dst, vec src2, kReg mask) %{
23998 match(Set dst (DivVF (Binary dst src2) mask));
23999 match(Set dst (DivVD (Binary dst src2) mask));
24000 format %{ "vpdiv_masked $dst, $dst, $src2, $mask\t! div masked operation" %}
24001 ins_encode %{
24002 int vlen_enc = vector_length_encoding(this);
24003 BasicType bt = Matcher::vector_element_basic_type(this);
24004 int opc = this->ideal_Opcode();
24005 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24006 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24007 %}
24008 ins_pipe( pipe_slow );
24009 %}
24010
24011 instruct vdiv_mem_masked(vec dst, memory src2, kReg mask) %{
24012 match(Set dst (DivVF (Binary dst (LoadVector src2)) mask));
24013 match(Set dst (DivVD (Binary dst (LoadVector src2)) mask));
24014 format %{ "vpdiv_masked $dst, $dst, $src2, $mask\t! div masked operation" %}
24015 ins_encode %{
24016 int vlen_enc = vector_length_encoding(this);
24017 BasicType bt = Matcher::vector_element_basic_type(this);
24018 int opc = this->ideal_Opcode();
24019 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24020 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24021 %}
24022 ins_pipe( pipe_slow );
24023 %}
24024
24025
24026 instruct vrol_imm_masked(vec dst, immI8 shift, kReg mask) %{
24027 match(Set dst (RotateLeftV (Binary dst shift) mask));
24028 match(Set dst (RotateRightV (Binary dst shift) mask));
24029 format %{ "vprotate_imm_masked $dst, $dst, $shift, $mask\t! rotate masked operation" %}
24030 ins_encode %{
24031 int vlen_enc = vector_length_encoding(this);
24032 BasicType bt = Matcher::vector_element_basic_type(this);
24033 int opc = this->ideal_Opcode();
24034 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24035 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24036 %}
24037 ins_pipe( pipe_slow );
24038 %}
24039
24040 instruct vrol_reg_masked(vec dst, vec src2, kReg mask) %{
24041 match(Set dst (RotateLeftV (Binary dst src2) mask));
24042 match(Set dst (RotateRightV (Binary dst src2) mask));
24043 format %{ "vrotate_masked $dst, $dst, $src2, $mask\t! rotate masked operation" %}
24044 ins_encode %{
24045 int vlen_enc = vector_length_encoding(this);
24046 BasicType bt = Matcher::vector_element_basic_type(this);
24047 int opc = this->ideal_Opcode();
24048 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24049 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24050 %}
24051 ins_pipe( pipe_slow );
24052 %}
24053
24054 instruct vlshift_imm_masked(vec dst, immI8 shift, kReg mask) %{
24055 match(Set dst (LShiftVS (Binary dst (LShiftCntV shift)) mask));
24056 match(Set dst (LShiftVI (Binary dst (LShiftCntV shift)) mask));
24057 match(Set dst (LShiftVL (Binary dst (LShiftCntV shift)) mask));
24058 format %{ "vplshift_imm_masked $dst, $dst, $shift, $mask\t! lshift masked operation" %}
24059 ins_encode %{
24060 int vlen_enc = vector_length_encoding(this);
24061 BasicType bt = Matcher::vector_element_basic_type(this);
24062 int opc = this->ideal_Opcode();
24063 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24064 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24065 %}
24066 ins_pipe( pipe_slow );
24067 %}
24068
24069 instruct vlshift_reg_masked(vec dst, vec src2, kReg mask) %{
24070 predicate(!n->as_ShiftV()->is_var_shift());
24071 match(Set dst (LShiftVS (Binary dst src2) mask));
24072 match(Set dst (LShiftVI (Binary dst src2) mask));
24073 match(Set dst (LShiftVL (Binary dst src2) mask));
24074 format %{ "vplshift_masked $dst, $dst, $src2, $mask\t! lshift masked operation" %}
24075 ins_encode %{
24076 int vlen_enc = vector_length_encoding(this);
24077 BasicType bt = Matcher::vector_element_basic_type(this);
24078 int opc = this->ideal_Opcode();
24079 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24080 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, false);
24081 %}
24082 ins_pipe( pipe_slow );
24083 %}
24084
24085 instruct vlshiftv_reg_masked(vec dst, vec src2, kReg mask) %{
24086 predicate(n->as_ShiftV()->is_var_shift());
24087 match(Set dst (LShiftVS (Binary dst src2) mask));
24088 match(Set dst (LShiftVI (Binary dst src2) mask));
24089 match(Set dst (LShiftVL (Binary dst src2) mask));
24090 format %{ "vplshiftv_masked $dst, $dst, $src2, $mask\t! lshift masked operation" %}
24091 ins_encode %{
24092 int vlen_enc = vector_length_encoding(this);
24093 BasicType bt = Matcher::vector_element_basic_type(this);
24094 int opc = this->ideal_Opcode();
24095 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24096 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, true);
24097 %}
24098 ins_pipe( pipe_slow );
24099 %}
24100
24101 instruct vrshift_imm_masked(vec dst, immI8 shift, kReg mask) %{
24102 match(Set dst (RShiftVS (Binary dst (RShiftCntV shift)) mask));
24103 match(Set dst (RShiftVI (Binary dst (RShiftCntV shift)) mask));
24104 match(Set dst (RShiftVL (Binary dst (RShiftCntV shift)) mask));
24105 format %{ "vprshift_imm_masked $dst, $dst, $shift, $mask\t! rshift masked operation" %}
24106 ins_encode %{
24107 int vlen_enc = vector_length_encoding(this);
24108 BasicType bt = Matcher::vector_element_basic_type(this);
24109 int opc = this->ideal_Opcode();
24110 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24111 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24112 %}
24113 ins_pipe( pipe_slow );
24114 %}
24115
24116 instruct vrshift_reg_masked(vec dst, vec src2, kReg mask) %{
24117 predicate(!n->as_ShiftV()->is_var_shift());
24118 match(Set dst (RShiftVS (Binary dst src2) mask));
24119 match(Set dst (RShiftVI (Binary dst src2) mask));
24120 match(Set dst (RShiftVL (Binary dst src2) mask));
24121 format %{ "vprshift_masked $dst, $dst, $src2, $mask\t! rshift masked operation" %}
24122 ins_encode %{
24123 int vlen_enc = vector_length_encoding(this);
24124 BasicType bt = Matcher::vector_element_basic_type(this);
24125 int opc = this->ideal_Opcode();
24126 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24127 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, false);
24128 %}
24129 ins_pipe( pipe_slow );
24130 %}
24131
24132 instruct vrshiftv_reg_masked(vec dst, vec src2, kReg mask) %{
24133 predicate(n->as_ShiftV()->is_var_shift());
24134 match(Set dst (RShiftVS (Binary dst src2) mask));
24135 match(Set dst (RShiftVI (Binary dst src2) mask));
24136 match(Set dst (RShiftVL (Binary dst src2) mask));
24137 format %{ "vprshiftv_masked $dst, $dst, $src2, $mask\t! rshift masked operation" %}
24138 ins_encode %{
24139 int vlen_enc = vector_length_encoding(this);
24140 BasicType bt = Matcher::vector_element_basic_type(this);
24141 int opc = this->ideal_Opcode();
24142 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24143 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, true);
24144 %}
24145 ins_pipe( pipe_slow );
24146 %}
24147
24148 instruct vurshift_imm_masked(vec dst, immI8 shift, kReg mask) %{
24149 match(Set dst (URShiftVS (Binary dst (RShiftCntV shift)) mask));
24150 match(Set dst (URShiftVI (Binary dst (RShiftCntV shift)) mask));
24151 match(Set dst (URShiftVL (Binary dst (RShiftCntV shift)) mask));
24152 format %{ "vpurshift_imm_masked $dst, $dst, $shift, $mask\t! urshift masked operation" %}
24153 ins_encode %{
24154 int vlen_enc = vector_length_encoding(this);
24155 BasicType bt = Matcher::vector_element_basic_type(this);
24156 int opc = this->ideal_Opcode();
24157 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24158 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24159 %}
24160 ins_pipe( pipe_slow );
24161 %}
24162
24163 instruct vurshift_reg_masked(vec dst, vec src2, kReg mask) %{
24164 predicate(!n->as_ShiftV()->is_var_shift());
24165 match(Set dst (URShiftVS (Binary dst src2) mask));
24166 match(Set dst (URShiftVI (Binary dst src2) mask));
24167 match(Set dst (URShiftVL (Binary dst src2) mask));
24168 format %{ "vpurshift_masked $dst, $dst, $src2, $mask\t! urshift masked operation" %}
24169 ins_encode %{
24170 int vlen_enc = vector_length_encoding(this);
24171 BasicType bt = Matcher::vector_element_basic_type(this);
24172 int opc = this->ideal_Opcode();
24173 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24174 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, false);
24175 %}
24176 ins_pipe( pipe_slow );
24177 %}
24178
24179 instruct vurshiftv_reg_masked(vec dst, vec src2, kReg mask) %{
24180 predicate(n->as_ShiftV()->is_var_shift());
24181 match(Set dst (URShiftVS (Binary dst src2) mask));
24182 match(Set dst (URShiftVI (Binary dst src2) mask));
24183 match(Set dst (URShiftVL (Binary dst src2) mask));
24184 format %{ "vpurshiftv_masked $dst, $dst, $src2, $mask\t! urshift masked operation" %}
24185 ins_encode %{
24186 int vlen_enc = vector_length_encoding(this);
24187 BasicType bt = Matcher::vector_element_basic_type(this);
24188 int opc = this->ideal_Opcode();
24189 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24190 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, true);
24191 %}
24192 ins_pipe( pipe_slow );
24193 %}
24194
24195 instruct vmaxv_reg_masked(vec dst, vec src2, kReg mask) %{
24196 match(Set dst (MaxV (Binary dst src2) mask));
24197 format %{ "vpmax_masked $dst, $dst, $src2, $mask\t! max masked operation" %}
24198 ins_encode %{
24199 int vlen_enc = vector_length_encoding(this);
24200 BasicType bt = Matcher::vector_element_basic_type(this);
24201 int opc = this->ideal_Opcode();
24202 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24203 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24204 %}
24205 ins_pipe( pipe_slow );
24206 %}
24207
24208 instruct vmaxv_mem_masked(vec dst, memory src2, kReg mask) %{
24209 match(Set dst (MaxV (Binary dst (LoadVector src2)) mask));
24210 format %{ "vpmax_masked $dst, $dst, $src2, $mask\t! max masked operation" %}
24211 ins_encode %{
24212 int vlen_enc = vector_length_encoding(this);
24213 BasicType bt = Matcher::vector_element_basic_type(this);
24214 int opc = this->ideal_Opcode();
24215 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24216 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24217 %}
24218 ins_pipe( pipe_slow );
24219 %}
24220
24221 instruct vminv_reg_masked(vec dst, vec src2, kReg mask) %{
24222 match(Set dst (MinV (Binary dst src2) mask));
24223 format %{ "vpmin_masked $dst, $dst, $src2, $mask\t! min masked operation" %}
24224 ins_encode %{
24225 int vlen_enc = vector_length_encoding(this);
24226 BasicType bt = Matcher::vector_element_basic_type(this);
24227 int opc = this->ideal_Opcode();
24228 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24229 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24230 %}
24231 ins_pipe( pipe_slow );
24232 %}
24233
24234 instruct vminv_mem_masked(vec dst, memory src2, kReg mask) %{
24235 match(Set dst (MinV (Binary dst (LoadVector src2)) mask));
24236 format %{ "vpmin_masked $dst, $dst, $src2, $mask\t! min masked operation" %}
24237 ins_encode %{
24238 int vlen_enc = vector_length_encoding(this);
24239 BasicType bt = Matcher::vector_element_basic_type(this);
24240 int opc = this->ideal_Opcode();
24241 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24242 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24243 %}
24244 ins_pipe( pipe_slow );
24245 %}
24246
24247 instruct vrearrangev_reg_masked(vec dst, vec src2, kReg mask) %{
24248 match(Set dst (VectorRearrange (Binary dst src2) mask));
24249 format %{ "vprearrange_masked $dst, $dst, $src2, $mask\t! rearrange masked operation" %}
24250 ins_encode %{
24251 int vlen_enc = vector_length_encoding(this);
24252 BasicType bt = Matcher::vector_element_basic_type(this);
24253 int opc = this->ideal_Opcode();
24254 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24255 $dst$$XMMRegister, $src2$$XMMRegister, false, vlen_enc);
24256 %}
24257 ins_pipe( pipe_slow );
24258 %}
24259
24260 instruct vabs_masked(vec dst, kReg mask) %{
24261 match(Set dst (AbsVB dst mask));
24262 match(Set dst (AbsVS dst mask));
24263 match(Set dst (AbsVI dst mask));
24264 match(Set dst (AbsVL dst mask));
24265 format %{ "vabs_masked $dst, $mask \t! vabs masked operation" %}
24266 ins_encode %{
24267 int vlen_enc = vector_length_encoding(this);
24268 BasicType bt = Matcher::vector_element_basic_type(this);
24269 int opc = this->ideal_Opcode();
24270 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24271 $dst$$XMMRegister, $dst$$XMMRegister, true, vlen_enc);
24272 %}
24273 ins_pipe( pipe_slow );
24274 %}
24275
24276 instruct vfma_reg_masked(vec dst, vec src2, vec src3, kReg mask) %{
24277 match(Set dst (FmaVF (Binary dst src2) (Binary src3 mask)));
24278 match(Set dst (FmaVD (Binary dst src2) (Binary src3 mask)));
24279 format %{ "vfma_masked $dst, $src2, $src3, $mask \t! vfma masked operation" %}
24280 ins_encode %{
24281 assert(UseFMA, "Needs FMA instructions support.");
24282 int vlen_enc = vector_length_encoding(this);
24283 BasicType bt = Matcher::vector_element_basic_type(this);
24284 int opc = this->ideal_Opcode();
24285 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24286 $src2$$XMMRegister, $src3$$XMMRegister, true, vlen_enc);
24287 %}
24288 ins_pipe( pipe_slow );
24289 %}
24290
24291 instruct vfma_mem_masked(vec dst, vec src2, memory src3, kReg mask) %{
24292 match(Set dst (FmaVF (Binary dst src2) (Binary (LoadVector src3) mask)));
24293 match(Set dst (FmaVD (Binary dst src2) (Binary (LoadVector src3) mask)));
24294 format %{ "vfma_masked $dst, $src2, $src3, $mask \t! vfma masked operation" %}
24295 ins_encode %{
24296 assert(UseFMA, "Needs FMA instructions support.");
24297 int vlen_enc = vector_length_encoding(this);
24298 BasicType bt = Matcher::vector_element_basic_type(this);
24299 int opc = this->ideal_Opcode();
24300 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24301 $src2$$XMMRegister, $src3$$Address, true, vlen_enc);
24302 %}
24303 ins_pipe( pipe_slow );
24304 %}
24305
24306 instruct evcmp_masked(kReg dst, vec src1, vec src2, immI8 cond, kReg mask) %{
24307 match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond mask)));
24308 format %{ "vcmp_masked $dst, $src1, $src2, $cond, $mask" %}
24309 ins_encode %{
24310 assert(bottom_type()->isa_pvectmask(), "TypePVectMask expected");
24311 int vlen_enc = vector_length_encoding(this, $src1);
24312 BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
24313
24314 // Comparison i
24315 switch (src1_elem_bt) {
24316 case T_BYTE: {
24317 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24318 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24319 __ evpcmpb($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24320 break;
24321 }
24322 case T_SHORT: {
24323 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24324 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24325 __ evpcmpw($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24326 break;
24327 }
24328 case T_INT: {
24329 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24330 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24331 __ evpcmpd($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24332 break;
24333 }
24334 case T_LONG: {
24335 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24336 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24337 __ evpcmpq($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24338 break;
24339 }
24340 case T_FLOAT: {
24341 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
24342 __ evcmpps($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
24343 break;
24344 }
24345 case T_DOUBLE: {
24346 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
24347 __ evcmppd($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
24348 break;
24349 }
24350 default: assert(false, "%s", type2name(src1_elem_bt)); break;
24351 }
24352 %}
24353 ins_pipe( pipe_slow );
24354 %}
24355
24356 instruct mask_all_evexI_LE32(kReg dst, rRegI src) %{
24357 predicate(Matcher::vector_length(n) <= 32);
24358 match(Set dst (MaskAll src));
24359 format %{ "mask_all_evexI_LE32 $dst, $src \t" %}
24360 ins_encode %{
24361 int mask_len = Matcher::vector_length(this);
24362 __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
24363 %}
24364 ins_pipe( pipe_slow );
24365 %}
24366
24367 instruct mask_not_immLT8(kReg dst, kReg src, rRegI rtmp, kReg ktmp, immI_M1 cnt) %{
24368 predicate(Matcher::vector_length(n) < 8 && VM_Version::supports_avx512dq());
24369 match(Set dst (XorVMask src (MaskAll cnt)));
24370 effect(TEMP_DEF dst, TEMP rtmp, TEMP ktmp);
24371 format %{ "mask_not_LT8 $dst, $src, $cnt \t!using $ktmp and $rtmp as TEMP" %}
24372 ins_encode %{
24373 uint masklen = Matcher::vector_length(this);
24374 __ knot(masklen, $dst$$KRegister, $src$$KRegister, $ktmp$$KRegister, $rtmp$$Register);
24375 %}
24376 ins_pipe( pipe_slow );
24377 %}
24378
24379 instruct mask_not_imm(kReg dst, kReg src, immI_M1 cnt) %{
24380 predicate((Matcher::vector_length(n) == 8 && VM_Version::supports_avx512dq()) ||
24381 (Matcher::vector_length(n) == 16) ||
24382 (Matcher::vector_length(n) > 16 && VM_Version::supports_avx512bw()));
24383 match(Set dst (XorVMask src (MaskAll cnt)));
24384 format %{ "mask_not $dst, $src, $cnt \t! mask not operation" %}
24385 ins_encode %{
24386 uint masklen = Matcher::vector_length(this);
24387 __ knot(masklen, $dst$$KRegister, $src$$KRegister);
24388 %}
24389 ins_pipe( pipe_slow );
24390 %}
24391
24392 instruct long_to_maskLE8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2) %{
24393 predicate(n->bottom_type()->isa_pvectmask() == nullptr && Matcher::vector_length(n) <= 8);
24394 match(Set dst (VectorLongToMask src));
24395 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2);
24396 format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2" %}
24397 ins_encode %{
24398 int mask_len = Matcher::vector_length(this);
24399 int vec_enc = vector_length_encoding(mask_len);
24400 __ vector_long_to_maskvec($dst$$XMMRegister, $src$$Register, $rtmp1$$Register,
24401 $rtmp2$$Register, xnoreg, mask_len, vec_enc);
24402 %}
24403 ins_pipe( pipe_slow );
24404 %}
24405
24406
24407 instruct long_to_maskGT8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec xtmp1, rFlagsReg cr) %{
24408 predicate(n->bottom_type()->isa_pvectmask() == nullptr && Matcher::vector_length(n) > 8);
24409 match(Set dst (VectorLongToMask src));
24410 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, TEMP xtmp1, KILL cr);
24411 format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2, $xtmp1, as TEMP" %}
24412 ins_encode %{
24413 int mask_len = Matcher::vector_length(this);
24414 assert(mask_len <= 32, "invalid mask length");
24415 int vec_enc = vector_length_encoding(mask_len);
24416 __ vector_long_to_maskvec($dst$$XMMRegister, $src$$Register, $rtmp1$$Register,
24417 $rtmp2$$Register, $xtmp1$$XMMRegister, mask_len, vec_enc);
24418 %}
24419 ins_pipe( pipe_slow );
24420 %}
24421
24422 instruct long_to_mask_evex(kReg dst, rRegL src) %{
24423 predicate(n->bottom_type()->isa_pvectmask());
24424 match(Set dst (VectorLongToMask src));
24425 format %{ "long_to_mask_evex $dst, $src\t!" %}
24426 ins_encode %{
24427 __ kmov($dst$$KRegister, $src$$Register);
24428 %}
24429 ins_pipe( pipe_slow );
24430 %}
24431
24432 instruct mask_opers_evex(kReg dst, kReg src1, kReg src2, kReg kscratch) %{
24433 match(Set dst (AndVMask src1 src2));
24434 match(Set dst (OrVMask src1 src2));
24435 match(Set dst (XorVMask src1 src2));
24436 effect(TEMP kscratch);
24437 format %{ "mask_opers_evex $dst, $src1, $src2\t! using $kscratch as TEMP" %}
24438 ins_encode %{
24439 const MachNode* mask1 = static_cast<const MachNode*>(this->in(this->operand_index($src1)));
24440 const MachNode* mask2 = static_cast<const MachNode*>(this->in(this->operand_index($src2)));
24441 assert(Type::equals(mask1->bottom_type(), mask2->bottom_type()), "Mask types must be equal");
24442 uint masklen = Matcher::vector_length(this);
24443 masklen = (masklen < 16 && !VM_Version::supports_avx512dq()) ? 16 : masklen;
24444 __ masked_op(this->ideal_Opcode(), masklen, $dst$$KRegister, $src1$$KRegister, $src2$$KRegister);
24445 %}
24446 ins_pipe( pipe_slow );
24447 %}
24448
24449 instruct vternlog_reg_masked(vec dst, vec src2, vec src3, immU8 func, kReg mask) %{
24450 match(Set dst (MacroLogicV dst (Binary src2 (Binary src3 (Binary func mask)))));
24451 format %{ "vternlog_masked $dst,$src2,$src3,$func,$mask\t! vternlog masked operation" %}
24452 ins_encode %{
24453 int vlen_enc = vector_length_encoding(this);
24454 BasicType bt = Matcher::vector_element_basic_type(this);
24455 __ evpternlog($dst$$XMMRegister, $func$$constant, $mask$$KRegister,
24456 $src2$$XMMRegister, $src3$$XMMRegister, true, bt, vlen_enc);
24457 %}
24458 ins_pipe( pipe_slow );
24459 %}
24460
24461 instruct vternlogd_mem_masked(vec dst, vec src2, memory src3, immU8 func, kReg mask) %{
24462 match(Set dst (MacroLogicV dst (Binary src2 (Binary src3 (Binary func mask)))));
24463 format %{ "vternlog_masked $dst,$src2,$src3,$func,$mask\t! vternlog masked operation" %}
24464 ins_encode %{
24465 int vlen_enc = vector_length_encoding(this);
24466 BasicType bt = Matcher::vector_element_basic_type(this);
24467 __ evpternlog($dst$$XMMRegister, $func$$constant, $mask$$KRegister,
24468 $src2$$XMMRegister, $src3$$Address, true, bt, vlen_enc);
24469 %}
24470 ins_pipe( pipe_slow );
24471 %}
24472
24473 instruct castMM(kReg dst)
24474 %{
24475 match(Set dst (CastVV dst));
24476
24477 size(0);
24478 format %{ "# castVV of $dst" %}
24479 ins_encode(/* empty encoding */);
24480 ins_cost(0);
24481 ins_pipe(empty);
24482 %}
24483
24484 instruct castVV(vec dst)
24485 %{
24486 match(Set dst (CastVV dst));
24487
24488 size(0);
24489 format %{ "# castVV of $dst" %}
24490 ins_encode(/* empty encoding */);
24491 ins_cost(0);
24492 ins_pipe(empty);
24493 %}
24494
24495 instruct castVVLeg(legVec dst)
24496 %{
24497 match(Set dst (CastVV dst));
24498
24499 size(0);
24500 format %{ "# castVV of $dst" %}
24501 ins_encode(/* empty encoding */);
24502 ins_cost(0);
24503 ins_pipe(empty);
24504 %}
24505
24506 instruct FloatClassCheck_reg_reg_vfpclass(rRegI dst, regF src, kReg ktmp, rFlagsReg cr)
24507 %{
24508 match(Set dst (IsInfiniteF src));
24509 effect(TEMP ktmp, KILL cr);
24510 format %{ "float_class_check $dst, $src" %}
24511 ins_encode %{
24512 __ vfpclassss($ktmp$$KRegister, $src$$XMMRegister, 0x18);
24513 __ kmovbl($dst$$Register, $ktmp$$KRegister);
24514 %}
24515 ins_pipe(pipe_slow);
24516 %}
24517
24518 instruct DoubleClassCheck_reg_reg_vfpclass(rRegI dst, regD src, kReg ktmp, rFlagsReg cr)
24519 %{
24520 match(Set dst (IsInfiniteD src));
24521 effect(TEMP ktmp, KILL cr);
24522 format %{ "double_class_check $dst, $src" %}
24523 ins_encode %{
24524 __ vfpclasssd($ktmp$$KRegister, $src$$XMMRegister, 0x18);
24525 __ kmovbl($dst$$Register, $ktmp$$KRegister);
24526 %}
24527 ins_pipe(pipe_slow);
24528 %}
24529
24530 instruct vector_addsub_saturating_subword_reg(vec dst, vec src1, vec src2)
24531 %{
24532 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24533 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
24534 match(Set dst (SaturatingAddV src1 src2));
24535 match(Set dst (SaturatingSubV src1 src2));
24536 format %{ "vector_addsub_saturating_subword $dst, $src1, $src2" %}
24537 ins_encode %{
24538 int vlen_enc = vector_length_encoding(this);
24539 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24540 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24541 $src1$$XMMRegister, $src2$$XMMRegister, false, vlen_enc);
24542 %}
24543 ins_pipe(pipe_slow);
24544 %}
24545
24546 instruct vector_addsub_saturating_unsigned_subword_reg(vec dst, vec src1, vec src2)
24547 %{
24548 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24549 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
24550 match(Set dst (SaturatingAddV src1 src2));
24551 match(Set dst (SaturatingSubV src1 src2));
24552 format %{ "vector_addsub_saturating_unsigned_subword $dst, $src1, $src2" %}
24553 ins_encode %{
24554 int vlen_enc = vector_length_encoding(this);
24555 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24556 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24557 $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24558 %}
24559 ins_pipe(pipe_slow);
24560 %}
24561
24562 instruct vector_addsub_saturating_reg_evex(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2)
24563 %{
24564 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24565 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned() &&
24566 (Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()));
24567 match(Set dst (SaturatingAddV src1 src2));
24568 match(Set dst (SaturatingSubV src1 src2));
24569 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2);
24570 format %{ "vector_addsub_saturating_evex $dst, $src1, $src2 \t! using $xtmp1, $xtmp2, $ktmp1 and $ktmp2 as TEMP" %}
24571 ins_encode %{
24572 int vlen_enc = vector_length_encoding(this);
24573 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24574 __ vector_addsub_dq_saturating_evex(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24575 $src1$$XMMRegister, $src2$$XMMRegister,
24576 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister,
24577 $ktmp1$$KRegister, $ktmp2$$KRegister, vlen_enc);
24578 %}
24579 ins_pipe(pipe_slow);
24580 %}
24581
24582 instruct vector_addsub_saturating_reg_avx(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4)
24583 %{
24584 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24585 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned() &&
24586 Matcher::vector_length_in_bytes(n) <= 32 && !VM_Version::supports_avx512vl());
24587 match(Set dst (SaturatingAddV src1 src2));
24588 match(Set dst (SaturatingSubV src1 src2));
24589 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4);
24590 format %{ "vector_addsub_saturating_avx $dst, $src1, $src2 \t! using $xtmp1, $xtmp2, $xtmp3 and $xtmp4 as TEMP" %}
24591 ins_encode %{
24592 int vlen_enc = vector_length_encoding(this);
24593 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24594 __ vector_addsub_dq_saturating_avx(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister, $src1$$XMMRegister,
24595 $src2$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister,
24596 $xtmp3$$XMMRegister, $xtmp4$$XMMRegister, vlen_enc);
24597 %}
24598 ins_pipe(pipe_slow);
24599 %}
24600
24601 instruct vector_add_saturating_unsigned_reg_evex(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, kReg ktmp)
24602 %{
24603 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24604 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24605 (Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()));
24606 match(Set dst (SaturatingAddV src1 src2));
24607 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp);
24608 format %{ "vector_add_saturating_unsigned_evex $dst, $src1, $src2 \t! using $xtmp1, $xtmp2 and $ktmp as TEMP" %}
24609 ins_encode %{
24610 int vlen_enc = vector_length_encoding(this);
24611 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24612 __ vector_add_dq_saturating_unsigned_evex(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
24613 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp$$KRegister, vlen_enc);
24614 %}
24615 ins_pipe(pipe_slow);
24616 %}
24617
24618 instruct vector_add_saturating_unsigned_reg_avx(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, vec xtmp3)
24619 %{
24620 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24621 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24622 Matcher::vector_length_in_bytes(n) <= 32 && !VM_Version::supports_avx512vl());
24623 match(Set dst (SaturatingAddV src1 src2));
24624 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3);
24625 format %{ "vector_add_saturating_unsigned_avx $dst, $src1, $src2 \t! using $xtmp1, $xtmp2 and $xtmp3 as TEMP" %}
24626 ins_encode %{
24627 int vlen_enc = vector_length_encoding(this);
24628 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24629 __ vector_add_dq_saturating_unsigned_avx(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
24630 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, vlen_enc);
24631 %}
24632 ins_pipe(pipe_slow);
24633 %}
24634
24635 instruct vector_sub_saturating_unsigned_reg_evex(vec dst, vec src1, vec src2, kReg ktmp)
24636 %{
24637 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24638 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24639 (Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()));
24640 match(Set dst (SaturatingSubV src1 src2));
24641 effect(TEMP ktmp);
24642 format %{ "vector_sub_saturating_unsigned_evex $dst, $src1, $src2 \t! using $ktmp as TEMP" %}
24643 ins_encode %{
24644 int vlen_enc = vector_length_encoding(this);
24645 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24646 __ vector_sub_dq_saturating_unsigned_evex(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister,
24647 $src2$$XMMRegister, $ktmp$$KRegister, vlen_enc);
24648 %}
24649 ins_pipe(pipe_slow);
24650 %}
24651
24652 instruct vector_sub_saturating_unsigned_reg_avx(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2)
24653 %{
24654 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24655 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24656 Matcher::vector_length_in_bytes(n) <= 32 && !VM_Version::supports_avx512vl());
24657 match(Set dst (SaturatingSubV src1 src2));
24658 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
24659 format %{ "vector_sub_saturating_unsigned_avx $dst, $src1, $src2 \t! using $xtmp1 and $xtmp2 as TEMP" %}
24660 ins_encode %{
24661 int vlen_enc = vector_length_encoding(this);
24662 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24663 __ vector_sub_dq_saturating_unsigned_avx(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
24664 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
24665 %}
24666 ins_pipe(pipe_slow);
24667 %}
24668
24669 instruct vector_addsub_saturating_subword_mem(vec dst, vec src1, memory src2)
24670 %{
24671 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24672 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
24673 match(Set dst (SaturatingAddV src1 (LoadVector src2)));
24674 match(Set dst (SaturatingSubV src1 (LoadVector src2)));
24675 format %{ "vector_addsub_saturating_subword $dst, $src1, $src2" %}
24676 ins_encode %{
24677 int vlen_enc = vector_length_encoding(this);
24678 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24679 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24680 $src1$$XMMRegister, $src2$$Address, false, vlen_enc);
24681 %}
24682 ins_pipe(pipe_slow);
24683 %}
24684
24685 instruct vector_addsub_saturating_unsigned_subword_mem(vec dst, vec src1, memory src2)
24686 %{
24687 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24688 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
24689 match(Set dst (SaturatingAddV src1 (LoadVector src2)));
24690 match(Set dst (SaturatingSubV src1 (LoadVector src2)));
24691 format %{ "vector_addsub_saturating_unsigned_subword $dst, $src1, $src2" %}
24692 ins_encode %{
24693 int vlen_enc = vector_length_encoding(this);
24694 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24695 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24696 $src1$$XMMRegister, $src2$$Address, true, vlen_enc);
24697 %}
24698 ins_pipe(pipe_slow);
24699 %}
24700
24701 instruct vector_addsub_saturating_subword_masked_reg(vec dst, vec src, kReg mask) %{
24702 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24703 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
24704 match(Set dst (SaturatingAddV (Binary dst src) mask));
24705 match(Set dst (SaturatingSubV (Binary dst src) mask));
24706 format %{ "vector_addsub_saturating_subword_masked $dst, $mask, $src" %}
24707 ins_encode %{
24708 int vlen_enc = vector_length_encoding(this);
24709 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24710 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
24711 $dst$$XMMRegister, $src$$XMMRegister, false, true, vlen_enc);
24712 %}
24713 ins_pipe( pipe_slow );
24714 %}
24715
24716 instruct vector_addsub_saturating_unsigned_subword_masked_reg(vec dst, vec src, kReg mask) %{
24717 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24718 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
24719 match(Set dst (SaturatingAddV (Binary dst src) mask));
24720 match(Set dst (SaturatingSubV (Binary dst src) mask));
24721 format %{ "vector_addsub_saturating_unsigned_subword_masked $dst, $mask, $src" %}
24722 ins_encode %{
24723 int vlen_enc = vector_length_encoding(this);
24724 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24725 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
24726 $dst$$XMMRegister, $src$$XMMRegister, true, true, vlen_enc);
24727 %}
24728 ins_pipe( pipe_slow );
24729 %}
24730
24731 instruct vector_addsub_saturating_subword_masked_mem(vec dst, memory src, kReg mask) %{
24732 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24733 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
24734 match(Set dst (SaturatingAddV (Binary dst (LoadVector src)) mask));
24735 match(Set dst (SaturatingSubV (Binary dst (LoadVector src)) mask));
24736 format %{ "vector_addsub_saturating_subword_masked $dst, $mask, $src" %}
24737 ins_encode %{
24738 int vlen_enc = vector_length_encoding(this);
24739 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24740 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
24741 $dst$$XMMRegister, $src$$Address, false, true, vlen_enc);
24742 %}
24743 ins_pipe( pipe_slow );
24744 %}
24745
24746 instruct vector_addsub_saturating_unsigned_subword_masked_mem(vec dst, memory src, kReg mask) %{
24747 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24748 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
24749 match(Set dst (SaturatingAddV (Binary dst (LoadVector src)) mask));
24750 match(Set dst (SaturatingSubV (Binary dst (LoadVector src)) mask));
24751 format %{ "vector_addsub_saturating_unsigned_subword_masked $dst, $mask, $src" %}
24752 ins_encode %{
24753 int vlen_enc = vector_length_encoding(this);
24754 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24755 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
24756 $dst$$XMMRegister, $src$$Address, true, true, vlen_enc);
24757 %}
24758 ins_pipe( pipe_slow );
24759 %}
24760
24761 instruct vector_selectfrom_twovectors_reg_evex(vec index, vec src1, vec src2)
24762 %{
24763 match(Set index (SelectFromTwoVector (Binary index src1) src2));
24764 format %{ "select_from_two_vector $index, $src1, $src2 \t!" %}
24765 ins_encode %{
24766 int vlen_enc = vector_length_encoding(this);
24767 BasicType bt = Matcher::vector_element_basic_type(this);
24768 __ select_from_two_vectors_evex(bt, $index$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
24769 %}
24770 ins_pipe(pipe_slow);
24771 %}
24772
24773 instruct reinterpretS2HF(regF dst, rRegI src)
24774 %{
24775 match(Set dst (ReinterpretS2HF src));
24776 format %{ "evmovw $dst, $src" %}
24777 ins_encode %{
24778 __ evmovw($dst$$XMMRegister, $src$$Register);
24779 %}
24780 ins_pipe(pipe_slow);
24781 %}
24782
24783 instruct reinterpretHF2S(rRegI dst, regF src)
24784 %{
24785 match(Set dst (ReinterpretHF2S src));
24786 format %{ "evmovw $dst, $src" %}
24787 ins_encode %{
24788 __ evmovw($dst$$Register, $src$$XMMRegister);
24789 __ narrow_subword_type($dst$$Register, T_SHORT);
24790 %}
24791 ins_pipe(pipe_slow);
24792 %}
24793
24794 instruct convF2HFAndS2HF(regF dst, regF src)
24795 %{
24796 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
24797 format %{ "convF2HFAndS2HF $dst, $src" %}
24798 ins_encode %{
24799 __ vcvtps2ph($dst$$XMMRegister, $src$$XMMRegister, 0x04, Assembler::AVX_128bit);
24800 %}
24801 ins_pipe(pipe_slow);
24802 %}
24803
24804 instruct convHF2SAndHF2F(regF dst, regF src)
24805 %{
24806 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
24807 format %{ "convHF2SAndHF2F $dst, $src" %}
24808 ins_encode %{
24809 __ vcvtph2ps($dst$$XMMRegister, $src$$XMMRegister, Assembler::AVX_128bit);
24810 %}
24811 ins_pipe(pipe_slow);
24812 %}
24813
24814 instruct scalar_sqrt_HF_reg(regF dst, regF src)
24815 %{
24816 match(Set dst (SqrtHF src));
24817 format %{ "scalar_sqrt_fp16 $dst, $src" %}
24818 ins_encode %{
24819 __ vsqrtsh($dst$$XMMRegister, $src$$XMMRegister);
24820 %}
24821 ins_pipe(pipe_slow);
24822 %}
24823
24824 instruct scalar_binOps_HF_reg(regF dst, regF src1, regF src2)
24825 %{
24826 match(Set dst (AddHF src1 src2));
24827 match(Set dst (DivHF src1 src2));
24828 match(Set dst (MulHF src1 src2));
24829 match(Set dst (SubHF src1 src2));
24830 format %{ "scalar_binop_fp16 $dst, $src1, $src2" %}
24831 ins_encode %{
24832 int opcode = this->ideal_Opcode();
24833 __ efp16sh(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
24834 %}
24835 ins_pipe(pipe_slow);
24836 %}
24837
24838 instruct scalar_minmax_HF_reg_avx10_2(regF dst, regF src1, regF src2)
24839 %{
24840 predicate(VM_Version::supports_avx10_2());
24841 match(Set dst (MaxHF src1 src2));
24842 match(Set dst (MinHF src1 src2));
24843
24844 format %{ "scalar_min_max_fp16 $dst, $src1, $src2" %}
24845 ins_encode %{
24846 int opcode = this->ideal_Opcode();
24847 __ sminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, k0);
24848 %}
24849 ins_pipe( pipe_slow );
24850 %}
24851
24852 instruct scalar_minmax_HF_reg(regF dst, regF src1, regF src2, kReg ktmp, regF xtmp1, regF xtmp2)
24853 %{
24854 predicate(!VM_Version::supports_avx10_2());
24855 match(Set dst (MaxHF src1 src2));
24856 match(Set dst (MinHF src1 src2));
24857 effect(TEMP_DEF dst, TEMP ktmp, TEMP xtmp1, TEMP xtmp2);
24858
24859 format %{ "scalar_min_max_fp16 $dst, $src1, $src2\t using $ktmp, $xtmp1 and $xtmp2 as TEMP" %}
24860 ins_encode %{
24861 int opcode = this->ideal_Opcode();
24862 __ sminmax_fp16(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $ktmp$$KRegister,
24863 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
24864 %}
24865 ins_pipe( pipe_slow );
24866 %}
24867
24868 instruct scalar_fma_HF_reg(regF dst, regF src1, regF src2)
24869 %{
24870 match(Set dst (FmaHF src2 (Binary dst src1)));
24871 effect(DEF dst);
24872 format %{ "scalar_fma_fp16 $dst, $src1, $src2\t# $dst = $dst * $src1 + $src2 fma packedH" %}
24873 ins_encode %{
24874 __ vfmadd132sh($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister);
24875 %}
24876 ins_pipe( pipe_slow );
24877 %}
24878
24879
24880 instruct vector_sqrt_HF_reg(vec dst, vec src)
24881 %{
24882 match(Set dst (SqrtVHF src));
24883 format %{ "vector_sqrt_fp16 $dst, $src" %}
24884 ins_encode %{
24885 int vlen_enc = vector_length_encoding(this);
24886 __ evsqrtph($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
24887 %}
24888 ins_pipe(pipe_slow);
24889 %}
24890
24891 instruct vector_sqrt_HF_mem(vec dst, memory src)
24892 %{
24893 match(Set dst (SqrtVHF (VectorReinterpret (LoadVector src))));
24894 format %{ "vector_sqrt_fp16_mem $dst, $src" %}
24895 ins_encode %{
24896 int vlen_enc = vector_length_encoding(this);
24897 __ evsqrtph($dst$$XMMRegister, $src$$Address, vlen_enc);
24898 %}
24899 ins_pipe(pipe_slow);
24900 %}
24901
24902 instruct vector_binOps_HF_reg(vec dst, vec src1, vec src2)
24903 %{
24904 match(Set dst (AddVHF src1 src2));
24905 match(Set dst (DivVHF src1 src2));
24906 match(Set dst (MulVHF src1 src2));
24907 match(Set dst (SubVHF src1 src2));
24908 format %{ "vector_binop_fp16 $dst, $src1, $src2" %}
24909 ins_encode %{
24910 int vlen_enc = vector_length_encoding(this);
24911 int opcode = this->ideal_Opcode();
24912 __ evfp16ph(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
24913 %}
24914 ins_pipe(pipe_slow);
24915 %}
24916
24917
24918 instruct vector_binOps_HF_mem(vec dst, vec src1, memory src2)
24919 %{
24920 match(Set dst (AddVHF src1 (VectorReinterpret (LoadVector src2))));
24921 match(Set dst (DivVHF src1 (VectorReinterpret (LoadVector src2))));
24922 match(Set dst (MulVHF src1 (VectorReinterpret (LoadVector src2))));
24923 match(Set dst (SubVHF src1 (VectorReinterpret (LoadVector src2))));
24924 format %{ "vector_binop_fp16_mem $dst, $src1, $src2" %}
24925 ins_encode %{
24926 int vlen_enc = vector_length_encoding(this);
24927 int opcode = this->ideal_Opcode();
24928 __ evfp16ph(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address, vlen_enc);
24929 %}
24930 ins_pipe(pipe_slow);
24931 %}
24932
24933 instruct vector_fma_HF_reg(vec dst, vec src1, vec src2)
24934 %{
24935 match(Set dst (FmaVHF src2 (Binary dst src1)));
24936 format %{ "vector_fma_fp16 $dst, $src1, $src2\t# $dst = $dst * $src1 + $src2 fma packedH" %}
24937 ins_encode %{
24938 int vlen_enc = vector_length_encoding(this);
24939 __ evfmadd132ph($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vlen_enc);
24940 %}
24941 ins_pipe( pipe_slow );
24942 %}
24943
24944 instruct vector_fma_HF_mem(vec dst, memory src1, vec src2)
24945 %{
24946 match(Set dst (FmaVHF src2 (Binary dst (VectorReinterpret (LoadVector src1)))));
24947 format %{ "vector_fma_fp16_mem $dst, $src1, $src2\t# $dst = $dst * $src1 + $src2 fma packedH" %}
24948 ins_encode %{
24949 int vlen_enc = vector_length_encoding(this);
24950 __ evfmadd132ph($dst$$XMMRegister, $src2$$XMMRegister, $src1$$Address, vlen_enc);
24951 %}
24952 ins_pipe( pipe_slow );
24953 %}
24954
24955 instruct vector_minmax_HF_mem_avx10_2(vec dst, vec src1, memory src2)
24956 %{
24957 predicate(VM_Version::supports_avx10_2());
24958 match(Set dst (MinVHF src1 (VectorReinterpret (LoadVector src2))));
24959 match(Set dst (MaxVHF src1 (VectorReinterpret (LoadVector src2))));
24960 format %{ "vector_min_max_fp16_mem $dst, $src1, $src2" %}
24961 ins_encode %{
24962 int vlen_enc = vector_length_encoding(this);
24963 int opcode = this->ideal_Opcode();
24964 __ vminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address,
24965 k0, vlen_enc);
24966 %}
24967 ins_pipe( pipe_slow );
24968 %}
24969
24970 instruct vector_minmax_HF_reg_avx10_2(vec dst, vec src1, vec src2)
24971 %{
24972 predicate(VM_Version::supports_avx10_2());
24973 match(Set dst (MinVHF src1 src2));
24974 match(Set dst (MaxVHF src1 src2));
24975 format %{ "vector_min_max_fp16 $dst, $src1, $src2" %}
24976 ins_encode %{
24977 int vlen_enc = vector_length_encoding(this);
24978 int opcode = this->ideal_Opcode();
24979 __ vminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
24980 k0, vlen_enc);
24981 %}
24982 ins_pipe( pipe_slow );
24983 %}
24984
24985 instruct vector_minmax_HF_reg(vec dst, vec src1, vec src2, kReg ktmp, vec xtmp1, vec xtmp2)
24986 %{
24987 predicate(!VM_Version::supports_avx10_2());
24988 match(Set dst (MinVHF src1 src2));
24989 match(Set dst (MaxVHF src1 src2));
24990 effect(TEMP_DEF dst, TEMP ktmp, TEMP xtmp1, TEMP xtmp2);
24991 format %{ "vector_min_max_fp16 $dst, $src1, $src2\t using $ktmp, $xtmp1 and $xtmp2 as TEMP" %}
24992 ins_encode %{
24993 int vlen_enc = vector_length_encoding(this);
24994 int opcode = this->ideal_Opcode();
24995 __ vminmax_fp16(opcode, $dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, $ktmp$$KRegister,
24996 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
24997 %}
24998 ins_pipe( pipe_slow );
24999 %}
25000
25001 //----------PEEPHOLE RULES-----------------------------------------------------
25002 // These must follow all instruction definitions as they use the names
25003 // defined in the instructions definitions.
25004 //
25005 // peeppredicate ( rule_predicate );
25006 // // the predicate unless which the peephole rule will be ignored
25007 //
25008 // peepmatch ( root_instr_name [preceding_instruction]* );
25009 //
25010 // peepprocedure ( procedure_name );
25011 // // provide a procedure name to perform the optimization, the procedure should
25012 // // reside in the architecture dependent peephole file, the method has the
25013 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
25014 // // with the arguments being the basic block, the current node index inside the
25015 // // block, the register allocator, the functions upon invoked return a new node
25016 // // defined in peepreplace, and the rules of the nodes appearing in the
25017 // // corresponding peepmatch, the function return true if successful, else
25018 // // return false
25019 //
25020 // peepconstraint %{
25021 // (instruction_number.operand_name relational_op instruction_number.operand_name
25022 // [, ...] );
25023 // // instruction numbers are zero-based using left to right order in peepmatch
25024 //
25025 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
25026 // // provide an instruction_number.operand_name for each operand that appears
25027 // // in the replacement instruction's match rule
25028 //
25029 // ---------VM FLAGS---------------------------------------------------------
25030 //
25031 // All peephole optimizations can be turned off using -XX:-OptoPeephole
25032 //
25033 // Each peephole rule is given an identifying number starting with zero and
25034 // increasing by one in the order seen by the parser. An individual peephole
25035 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
25036 // on the command-line.
25037 //
25038 // ---------CURRENT LIMITATIONS----------------------------------------------
25039 //
25040 // Only transformations inside a basic block (do we need more for peephole)
25041 //
25042 // ---------EXAMPLE----------------------------------------------------------
25043 //
25044 // // pertinent parts of existing instructions in architecture description
25045 // instruct movI(rRegI dst, rRegI src)
25046 // %{
25047 // match(Set dst (CopyI src));
25048 // %}
25049 //
25050 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
25051 // %{
25052 // match(Set dst (AddI dst src));
25053 // effect(KILL cr);
25054 // %}
25055 //
25056 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
25057 // %{
25058 // match(Set dst (AddI dst src));
25059 // %}
25060 //
25061 // 1. Simple replacement
25062 // - Only match adjacent instructions in same basic block
25063 // - Only equality constraints
25064 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
25065 // - Only one replacement instruction
25066 //
25067 // // Change (inc mov) to lea
25068 // peephole %{
25069 // // lea should only be emitted when beneficial
25070 // peeppredicate( VM_Version::supports_fast_2op_lea() );
25071 // // increment preceded by register-register move
25072 // peepmatch ( incI_rReg movI );
25073 // // require that the destination register of the increment
25074 // // match the destination register of the move
25075 // peepconstraint ( 0.dst == 1.dst );
25076 // // construct a replacement instruction that sets
25077 // // the destination to ( move's source register + one )
25078 // peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
25079 // %}
25080 //
25081 // 2. Procedural replacement
25082 // - More flexible finding relevent nodes
25083 // - More flexible constraints
25084 // - More flexible transformations
25085 // - May utilise architecture-dependent API more effectively
25086 // - Currently only one replacement instruction due to adlc parsing capabilities
25087 //
25088 // // Change (inc mov) to lea
25089 // peephole %{
25090 // // lea should only be emitted when beneficial
25091 // peeppredicate( VM_Version::supports_fast_2op_lea() );
25092 // // the rule numbers of these nodes inside are passed into the function below
25093 // peepmatch ( incI_rReg movI );
25094 // // the method that takes the responsibility of transformation
25095 // peepprocedure ( inc_mov_to_lea );
25096 // // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
25097 // // node is passed into the function above
25098 // peepreplace ( leaI_rReg_immI() );
25099 // %}
25100
25101 // These instructions is not matched by the matcher but used by the peephole
25102 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
25103 %{
25104 predicate(false);
25105 match(Set dst (AddI src1 src2));
25106 format %{ "leal $dst, [$src1 + $src2]" %}
25107 ins_encode %{
25108 Register dst = $dst$$Register;
25109 Register src1 = $src1$$Register;
25110 Register src2 = $src2$$Register;
25111 if (src1 != rbp && src1 != r13) {
25112 __ leal(dst, Address(src1, src2, Address::times_1));
25113 } else {
25114 assert(src2 != rbp && src2 != r13, "");
25115 __ leal(dst, Address(src2, src1, Address::times_1));
25116 }
25117 %}
25118 ins_pipe(ialu_reg_reg);
25119 %}
25120
25121 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
25122 %{
25123 predicate(false);
25124 match(Set dst (AddI src1 src2));
25125 format %{ "leal $dst, [$src1 + $src2]" %}
25126 ins_encode %{
25127 __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
25128 %}
25129 ins_pipe(ialu_reg_reg);
25130 %}
25131
25132 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
25133 %{
25134 predicate(false);
25135 match(Set dst (LShiftI src shift));
25136 format %{ "leal $dst, [$src << $shift]" %}
25137 ins_encode %{
25138 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
25139 Register src = $src$$Register;
25140 if (scale == Address::times_2 && src != rbp && src != r13) {
25141 __ leal($dst$$Register, Address(src, src, Address::times_1));
25142 } else {
25143 __ leal($dst$$Register, Address(noreg, src, scale));
25144 }
25145 %}
25146 ins_pipe(ialu_reg_reg);
25147 %}
25148
25149 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
25150 %{
25151 predicate(false);
25152 match(Set dst (AddL src1 src2));
25153 format %{ "leaq $dst, [$src1 + $src2]" %}
25154 ins_encode %{
25155 Register dst = $dst$$Register;
25156 Register src1 = $src1$$Register;
25157 Register src2 = $src2$$Register;
25158 if (src1 != rbp && src1 != r13) {
25159 __ leaq(dst, Address(src1, src2, Address::times_1));
25160 } else {
25161 assert(src2 != rbp && src2 != r13, "");
25162 __ leaq(dst, Address(src2, src1, Address::times_1));
25163 }
25164 %}
25165 ins_pipe(ialu_reg_reg);
25166 %}
25167
25168 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
25169 %{
25170 predicate(false);
25171 match(Set dst (AddL src1 src2));
25172 format %{ "leaq $dst, [$src1 + $src2]" %}
25173 ins_encode %{
25174 __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
25175 %}
25176 ins_pipe(ialu_reg_reg);
25177 %}
25178
25179 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
25180 %{
25181 predicate(false);
25182 match(Set dst (LShiftL src shift));
25183 format %{ "leaq $dst, [$src << $shift]" %}
25184 ins_encode %{
25185 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
25186 Register src = $src$$Register;
25187 if (scale == Address::times_2 && src != rbp && src != r13) {
25188 __ leaq($dst$$Register, Address(src, src, Address::times_1));
25189 } else {
25190 __ leaq($dst$$Register, Address(noreg, src, scale));
25191 }
25192 %}
25193 ins_pipe(ialu_reg_reg);
25194 %}
25195
25196 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
25197 // sal}) with lea instructions. The {add, sal} rules are beneficial in
25198 // processors with at least partial ALU support for lea
25199 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
25200 // beneficial for processors with full ALU support
25201 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
25202
25203 peephole
25204 %{
25205 peeppredicate(VM_Version::supports_fast_2op_lea());
25206 peepmatch (addI_rReg);
25207 peepprocedure (lea_coalesce_reg);
25208 peepreplace (leaI_rReg_rReg_peep());
25209 %}
25210
25211 peephole
25212 %{
25213 peeppredicate(VM_Version::supports_fast_2op_lea());
25214 peepmatch (addI_rReg_imm);
25215 peepprocedure (lea_coalesce_imm);
25216 peepreplace (leaI_rReg_immI_peep());
25217 %}
25218
25219 peephole
25220 %{
25221 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25222 VM_Version::is_intel_cascade_lake());
25223 peepmatch (incI_rReg);
25224 peepprocedure (lea_coalesce_imm);
25225 peepreplace (leaI_rReg_immI_peep());
25226 %}
25227
25228 peephole
25229 %{
25230 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25231 VM_Version::is_intel_cascade_lake());
25232 peepmatch (decI_rReg);
25233 peepprocedure (lea_coalesce_imm);
25234 peepreplace (leaI_rReg_immI_peep());
25235 %}
25236
25237 peephole
25238 %{
25239 peeppredicate(VM_Version::supports_fast_2op_lea());
25240 peepmatch (salI_rReg_immI2);
25241 peepprocedure (lea_coalesce_imm);
25242 peepreplace (leaI_rReg_immI2_peep());
25243 %}
25244
25245 peephole
25246 %{
25247 peeppredicate(VM_Version::supports_fast_2op_lea());
25248 peepmatch (addL_rReg);
25249 peepprocedure (lea_coalesce_reg);
25250 peepreplace (leaL_rReg_rReg_peep());
25251 %}
25252
25253 peephole
25254 %{
25255 peeppredicate(VM_Version::supports_fast_2op_lea());
25256 peepmatch (addL_rReg_imm);
25257 peepprocedure (lea_coalesce_imm);
25258 peepreplace (leaL_rReg_immL32_peep());
25259 %}
25260
25261 peephole
25262 %{
25263 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25264 VM_Version::is_intel_cascade_lake());
25265 peepmatch (incL_rReg);
25266 peepprocedure (lea_coalesce_imm);
25267 peepreplace (leaL_rReg_immL32_peep());
25268 %}
25269
25270 peephole
25271 %{
25272 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25273 VM_Version::is_intel_cascade_lake());
25274 peepmatch (decL_rReg);
25275 peepprocedure (lea_coalesce_imm);
25276 peepreplace (leaL_rReg_immL32_peep());
25277 %}
25278
25279 peephole
25280 %{
25281 peeppredicate(VM_Version::supports_fast_2op_lea());
25282 peepmatch (salL_rReg_immI2);
25283 peepprocedure (lea_coalesce_imm);
25284 peepreplace (leaL_rReg_immI2_peep());
25285 %}
25286
25287 peephole
25288 %{
25289 peepmatch (leaPCompressedOopOffset);
25290 peepprocedure (lea_remove_redundant);
25291 %}
25292
25293 peephole
25294 %{
25295 peepmatch (leaP8Narrow);
25296 peepprocedure (lea_remove_redundant);
25297 %}
25298
25299 peephole
25300 %{
25301 peepmatch (leaP32Narrow);
25302 peepprocedure (lea_remove_redundant);
25303 %}
25304
25305 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
25306 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
25307
25308 //int variant
25309 peephole
25310 %{
25311 peepmatch (testI_reg);
25312 peepprocedure (test_may_remove);
25313 %}
25314
25315 //long variant
25316 peephole
25317 %{
25318 peepmatch (testL_reg);
25319 peepprocedure (test_may_remove);
25320 %}
25321
25322
25323 //----------SMARTSPILL RULES---------------------------------------------------
25324 // These must follow all instruction definitions as they use the names
25325 // defined in the instructions definitions.