1 //
2 // Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23 //
24
25 // X86 AMD64 Architecture Description File
26
27 //----------REGISTER DEFINITION BLOCK------------------------------------------
28 // This information is used by the matcher and the register allocator to
29 // describe individual registers and classes of registers within the target
30 // architecture.
31
32 register %{
33 //----------Architecture Description Register Definitions----------------------
34 // General Registers
35 // "reg_def" name ( register save type, C convention save type,
36 // ideal register type, encoding );
37 // Register Save Types:
38 //
39 // NS = No-Save: The register allocator assumes that these registers
40 // can be used without saving upon entry to the method, &
41 // that they do not need to be saved at call sites.
42 //
43 // SOC = Save-On-Call: The register allocator assumes that these registers
44 // can be used without saving upon entry to the method,
45 // but that they must be saved at call sites.
46 //
47 // SOE = Save-On-Entry: The register allocator assumes that these registers
48 // must be saved before using them upon entry to the
49 // method, but they do not need to be saved at call
50 // sites.
51 //
52 // AS = Always-Save: The register allocator assumes that these registers
53 // must be saved before using them upon entry to the
54 // method, & that they must be saved at call sites.
55 //
56 // Ideal Register Type is used to determine how to save & restore a
57 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
58 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
59 //
60 // The encoding number is the actual bit-pattern placed into the opcodes.
61
62 // General Registers
63 // R8-R15 must be encoded with REX. (RSP, RBP, RSI, RDI need REX when
64 // used as byte registers)
65
66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
69
70 reg_def RAX (SOC, SOC, Op_RegI, 0, rax->as_VMReg());
71 reg_def RAX_H(SOC, SOC, Op_RegI, 0, rax->as_VMReg()->next());
72
73 reg_def RCX (SOC, SOC, Op_RegI, 1, rcx->as_VMReg());
74 reg_def RCX_H(SOC, SOC, Op_RegI, 1, rcx->as_VMReg()->next());
75
76 reg_def RDX (SOC, SOC, Op_RegI, 2, rdx->as_VMReg());
77 reg_def RDX_H(SOC, SOC, Op_RegI, 2, rdx->as_VMReg()->next());
78
79 reg_def RBX (SOC, SOE, Op_RegI, 3, rbx->as_VMReg());
80 reg_def RBX_H(SOC, SOE, Op_RegI, 3, rbx->as_VMReg()->next());
81
82 reg_def RSP (NS, NS, Op_RegI, 4, rsp->as_VMReg());
83 reg_def RSP_H(NS, NS, Op_RegI, 4, rsp->as_VMReg()->next());
84
85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
86 reg_def RBP (NS, SOE, Op_RegI, 5, rbp->as_VMReg());
87 reg_def RBP_H(NS, SOE, Op_RegI, 5, rbp->as_VMReg()->next());
88
89 #ifdef _WIN64
90
91 reg_def RSI (SOC, SOE, Op_RegI, 6, rsi->as_VMReg());
92 reg_def RSI_H(SOC, SOE, Op_RegI, 6, rsi->as_VMReg()->next());
93
94 reg_def RDI (SOC, SOE, Op_RegI, 7, rdi->as_VMReg());
95 reg_def RDI_H(SOC, SOE, Op_RegI, 7, rdi->as_VMReg()->next());
96
97 #else
98
99 reg_def RSI (SOC, SOC, Op_RegI, 6, rsi->as_VMReg());
100 reg_def RSI_H(SOC, SOC, Op_RegI, 6, rsi->as_VMReg()->next());
101
102 reg_def RDI (SOC, SOC, Op_RegI, 7, rdi->as_VMReg());
103 reg_def RDI_H(SOC, SOC, Op_RegI, 7, rdi->as_VMReg()->next());
104
105 #endif
106
107 reg_def R8 (SOC, SOC, Op_RegI, 8, r8->as_VMReg());
108 reg_def R8_H (SOC, SOC, Op_RegI, 8, r8->as_VMReg()->next());
109
110 reg_def R9 (SOC, SOC, Op_RegI, 9, r9->as_VMReg());
111 reg_def R9_H (SOC, SOC, Op_RegI, 9, r9->as_VMReg()->next());
112
113 reg_def R10 (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
115
116 reg_def R11 (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
118
119 reg_def R12 (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
121
122 reg_def R13 (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
124
125 reg_def R14 (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
127
128 reg_def R15 (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
130
131 reg_def R16 (SOC, SOC, Op_RegI, 16, r16->as_VMReg());
132 reg_def R16_H(SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
133
134 reg_def R17 (SOC, SOC, Op_RegI, 17, r17->as_VMReg());
135 reg_def R17_H(SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
136
137 reg_def R18 (SOC, SOC, Op_RegI, 18, r18->as_VMReg());
138 reg_def R18_H(SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
139
140 reg_def R19 (SOC, SOC, Op_RegI, 19, r19->as_VMReg());
141 reg_def R19_H(SOC, SOC, Op_RegI, 19, r19->as_VMReg()->next());
142
143 reg_def R20 (SOC, SOC, Op_RegI, 20, r20->as_VMReg());
144 reg_def R20_H(SOC, SOC, Op_RegI, 20, r20->as_VMReg()->next());
145
146 reg_def R21 (SOC, SOC, Op_RegI, 21, r21->as_VMReg());
147 reg_def R21_H(SOC, SOC, Op_RegI, 21, r21->as_VMReg()->next());
148
149 reg_def R22 (SOC, SOC, Op_RegI, 22, r22->as_VMReg());
150 reg_def R22_H(SOC, SOC, Op_RegI, 22, r22->as_VMReg()->next());
151
152 reg_def R23 (SOC, SOC, Op_RegI, 23, r23->as_VMReg());
153 reg_def R23_H(SOC, SOC, Op_RegI, 23, r23->as_VMReg()->next());
154
155 reg_def R24 (SOC, SOC, Op_RegI, 24, r24->as_VMReg());
156 reg_def R24_H(SOC, SOC, Op_RegI, 24, r24->as_VMReg()->next());
157
158 reg_def R25 (SOC, SOC, Op_RegI, 25, r25->as_VMReg());
159 reg_def R25_H(SOC, SOC, Op_RegI, 25, r25->as_VMReg()->next());
160
161 reg_def R26 (SOC, SOC, Op_RegI, 26, r26->as_VMReg());
162 reg_def R26_H(SOC, SOC, Op_RegI, 26, r26->as_VMReg()->next());
163
164 reg_def R27 (SOC, SOC, Op_RegI, 27, r27->as_VMReg());
165 reg_def R27_H(SOC, SOC, Op_RegI, 27, r27->as_VMReg()->next());
166
167 reg_def R28 (SOC, SOC, Op_RegI, 28, r28->as_VMReg());
168 reg_def R28_H(SOC, SOC, Op_RegI, 28, r28->as_VMReg()->next());
169
170 reg_def R29 (SOC, SOC, Op_RegI, 29, r29->as_VMReg());
171 reg_def R29_H(SOC, SOC, Op_RegI, 29, r29->as_VMReg()->next());
172
173 reg_def R30 (SOC, SOC, Op_RegI, 30, r30->as_VMReg());
174 reg_def R30_H(SOC, SOC, Op_RegI, 30, r30->as_VMReg()->next());
175
176 reg_def R31 (SOC, SOC, Op_RegI, 31, r31->as_VMReg());
177 reg_def R31_H(SOC, SOC, Op_RegI, 31, r31->as_VMReg()->next());
178
179 // Floating Point Registers
180
181 // Specify priority of register selection within phases of register
182 // allocation. Highest priority is first. A useful heuristic is to
183 // give registers a low priority when they are required by machine
184 // instructions, like EAX and EDX on I486, and choose no-save registers
185 // before save-on-call, & save-on-call before save-on-entry. Registers
186 // which participate in fixed calling sequences should come last.
187 // Registers which are used as pairs must fall on an even boundary.
188
189 alloc_class chunk0(R10, R10_H,
190 R11, R11_H,
191 R8, R8_H,
192 R9, R9_H,
193 R12, R12_H,
194 RCX, RCX_H,
195 RBX, RBX_H,
196 RDI, RDI_H,
197 RDX, RDX_H,
198 RSI, RSI_H,
199 RAX, RAX_H,
200 RBP, RBP_H,
201 R13, R13_H,
202 R14, R14_H,
203 R15, R15_H,
204 R16, R16_H,
205 R17, R17_H,
206 R18, R18_H,
207 R19, R19_H,
208 R20, R20_H,
209 R21, R21_H,
210 R22, R22_H,
211 R23, R23_H,
212 R24, R24_H,
213 R25, R25_H,
214 R26, R26_H,
215 R27, R27_H,
216 R28, R28_H,
217 R29, R29_H,
218 R30, R30_H,
219 R31, R31_H,
220 RSP, RSP_H);
221
222 // XMM registers. 512-bit registers or 8 words each, labeled (a)-p.
223 // Word a in each register holds a Float, words ab hold a Double.
224 // The whole registers are used in SSE4.2 version intrinsics,
225 // array copy stubs and superword operations (see UseSSE42Intrinsics,
226 // UseXMMForArrayCopy and UseSuperword flags).
227 // For pre EVEX enabled architectures:
228 // XMM8-XMM15 must be encoded with REX (VEX for UseAVX)
229 // For EVEX enabled architectures:
230 // XMM8-XMM31 must be encoded with REX (EVEX for UseAVX).
231 //
232 // Linux ABI: No register preserved across function calls
233 // XMM0-XMM7 might hold parameters
234 // Windows ABI: XMM6-XMM15 preserved across function calls
235 // XMM0-XMM3 might hold parameters
236
237 reg_def XMM0 ( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg());
238 reg_def XMM0b( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(1));
239 reg_def XMM0c( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(2));
240 reg_def XMM0d( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(3));
241 reg_def XMM0e( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(4));
242 reg_def XMM0f( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(5));
243 reg_def XMM0g( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(6));
244 reg_def XMM0h( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(7));
245 reg_def XMM0i( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(8));
246 reg_def XMM0j( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(9));
247 reg_def XMM0k( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(10));
248 reg_def XMM0l( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(11));
249 reg_def XMM0m( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(12));
250 reg_def XMM0n( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(13));
251 reg_def XMM0o( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(14));
252 reg_def XMM0p( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(15));
253
254 reg_def XMM1 ( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg());
255 reg_def XMM1b( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(1));
256 reg_def XMM1c( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(2));
257 reg_def XMM1d( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(3));
258 reg_def XMM1e( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(4));
259 reg_def XMM1f( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(5));
260 reg_def XMM1g( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(6));
261 reg_def XMM1h( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(7));
262 reg_def XMM1i( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(8));
263 reg_def XMM1j( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(9));
264 reg_def XMM1k( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(10));
265 reg_def XMM1l( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(11));
266 reg_def XMM1m( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(12));
267 reg_def XMM1n( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(13));
268 reg_def XMM1o( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(14));
269 reg_def XMM1p( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(15));
270
271 reg_def XMM2 ( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg());
272 reg_def XMM2b( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(1));
273 reg_def XMM2c( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(2));
274 reg_def XMM2d( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(3));
275 reg_def XMM2e( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(4));
276 reg_def XMM2f( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(5));
277 reg_def XMM2g( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(6));
278 reg_def XMM2h( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(7));
279 reg_def XMM2i( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(8));
280 reg_def XMM2j( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(9));
281 reg_def XMM2k( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(10));
282 reg_def XMM2l( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(11));
283 reg_def XMM2m( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(12));
284 reg_def XMM2n( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(13));
285 reg_def XMM2o( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(14));
286 reg_def XMM2p( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(15));
287
288 reg_def XMM3 ( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg());
289 reg_def XMM3b( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(1));
290 reg_def XMM3c( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(2));
291 reg_def XMM3d( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(3));
292 reg_def XMM3e( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(4));
293 reg_def XMM3f( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(5));
294 reg_def XMM3g( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(6));
295 reg_def XMM3h( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(7));
296 reg_def XMM3i( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(8));
297 reg_def XMM3j( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(9));
298 reg_def XMM3k( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(10));
299 reg_def XMM3l( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(11));
300 reg_def XMM3m( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(12));
301 reg_def XMM3n( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(13));
302 reg_def XMM3o( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(14));
303 reg_def XMM3p( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(15));
304
305 reg_def XMM4 ( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg());
306 reg_def XMM4b( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(1));
307 reg_def XMM4c( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(2));
308 reg_def XMM4d( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(3));
309 reg_def XMM4e( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(4));
310 reg_def XMM4f( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(5));
311 reg_def XMM4g( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(6));
312 reg_def XMM4h( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(7));
313 reg_def XMM4i( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(8));
314 reg_def XMM4j( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(9));
315 reg_def XMM4k( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(10));
316 reg_def XMM4l( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(11));
317 reg_def XMM4m( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(12));
318 reg_def XMM4n( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(13));
319 reg_def XMM4o( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(14));
320 reg_def XMM4p( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(15));
321
322 reg_def XMM5 ( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg());
323 reg_def XMM5b( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(1));
324 reg_def XMM5c( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(2));
325 reg_def XMM5d( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(3));
326 reg_def XMM5e( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(4));
327 reg_def XMM5f( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(5));
328 reg_def XMM5g( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(6));
329 reg_def XMM5h( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(7));
330 reg_def XMM5i( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(8));
331 reg_def XMM5j( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(9));
332 reg_def XMM5k( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(10));
333 reg_def XMM5l( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(11));
334 reg_def XMM5m( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(12));
335 reg_def XMM5n( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(13));
336 reg_def XMM5o( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(14));
337 reg_def XMM5p( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(15));
338
339 reg_def XMM6 ( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg());
340 reg_def XMM6b( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(1));
341 reg_def XMM6c( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(2));
342 reg_def XMM6d( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(3));
343 reg_def XMM6e( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(4));
344 reg_def XMM6f( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(5));
345 reg_def XMM6g( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(6));
346 reg_def XMM6h( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(7));
347 reg_def XMM6i( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(8));
348 reg_def XMM6j( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(9));
349 reg_def XMM6k( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(10));
350 reg_def XMM6l( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(11));
351 reg_def XMM6m( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(12));
352 reg_def XMM6n( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(13));
353 reg_def XMM6o( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(14));
354 reg_def XMM6p( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(15));
355
356 reg_def XMM7 ( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg());
357 reg_def XMM7b( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(1));
358 reg_def XMM7c( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(2));
359 reg_def XMM7d( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(3));
360 reg_def XMM7e( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(4));
361 reg_def XMM7f( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(5));
362 reg_def XMM7g( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(6));
363 reg_def XMM7h( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(7));
364 reg_def XMM7i( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(8));
365 reg_def XMM7j( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(9));
366 reg_def XMM7k( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(10));
367 reg_def XMM7l( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(11));
368 reg_def XMM7m( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(12));
369 reg_def XMM7n( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(13));
370 reg_def XMM7o( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(14));
371 reg_def XMM7p( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(15));
372
373 reg_def XMM8 ( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg());
374 reg_def XMM8b( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(1));
375 reg_def XMM8c( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(2));
376 reg_def XMM8d( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(3));
377 reg_def XMM8e( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(4));
378 reg_def XMM8f( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(5));
379 reg_def XMM8g( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(6));
380 reg_def XMM8h( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(7));
381 reg_def XMM8i( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(8));
382 reg_def XMM8j( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(9));
383 reg_def XMM8k( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(10));
384 reg_def XMM8l( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(11));
385 reg_def XMM8m( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(12));
386 reg_def XMM8n( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(13));
387 reg_def XMM8o( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(14));
388 reg_def XMM8p( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(15));
389
390 reg_def XMM9 ( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg());
391 reg_def XMM9b( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(1));
392 reg_def XMM9c( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(2));
393 reg_def XMM9d( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(3));
394 reg_def XMM9e( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(4));
395 reg_def XMM9f( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(5));
396 reg_def XMM9g( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(6));
397 reg_def XMM9h( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(7));
398 reg_def XMM9i( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(8));
399 reg_def XMM9j( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(9));
400 reg_def XMM9k( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(10));
401 reg_def XMM9l( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(11));
402 reg_def XMM9m( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(12));
403 reg_def XMM9n( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(13));
404 reg_def XMM9o( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(14));
405 reg_def XMM9p( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(15));
406
407 reg_def XMM10 ( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg());
408 reg_def XMM10b( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(1));
409 reg_def XMM10c( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(2));
410 reg_def XMM10d( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(3));
411 reg_def XMM10e( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(4));
412 reg_def XMM10f( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(5));
413 reg_def XMM10g( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(6));
414 reg_def XMM10h( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(7));
415 reg_def XMM10i( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(8));
416 reg_def XMM10j( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(9));
417 reg_def XMM10k( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(10));
418 reg_def XMM10l( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(11));
419 reg_def XMM10m( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(12));
420 reg_def XMM10n( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(13));
421 reg_def XMM10o( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(14));
422 reg_def XMM10p( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(15));
423
424 reg_def XMM11 ( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg());
425 reg_def XMM11b( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(1));
426 reg_def XMM11c( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(2));
427 reg_def XMM11d( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(3));
428 reg_def XMM11e( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(4));
429 reg_def XMM11f( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(5));
430 reg_def XMM11g( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(6));
431 reg_def XMM11h( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(7));
432 reg_def XMM11i( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(8));
433 reg_def XMM11j( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(9));
434 reg_def XMM11k( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(10));
435 reg_def XMM11l( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(11));
436 reg_def XMM11m( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(12));
437 reg_def XMM11n( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(13));
438 reg_def XMM11o( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(14));
439 reg_def XMM11p( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(15));
440
441 reg_def XMM12 ( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg());
442 reg_def XMM12b( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(1));
443 reg_def XMM12c( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(2));
444 reg_def XMM12d( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(3));
445 reg_def XMM12e( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(4));
446 reg_def XMM12f( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(5));
447 reg_def XMM12g( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(6));
448 reg_def XMM12h( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(7));
449 reg_def XMM12i( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(8));
450 reg_def XMM12j( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(9));
451 reg_def XMM12k( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(10));
452 reg_def XMM12l( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(11));
453 reg_def XMM12m( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(12));
454 reg_def XMM12n( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(13));
455 reg_def XMM12o( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(14));
456 reg_def XMM12p( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(15));
457
458 reg_def XMM13 ( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg());
459 reg_def XMM13b( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(1));
460 reg_def XMM13c( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(2));
461 reg_def XMM13d( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(3));
462 reg_def XMM13e( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(4));
463 reg_def XMM13f( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(5));
464 reg_def XMM13g( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(6));
465 reg_def XMM13h( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(7));
466 reg_def XMM13i( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(8));
467 reg_def XMM13j( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(9));
468 reg_def XMM13k( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(10));
469 reg_def XMM13l( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(11));
470 reg_def XMM13m( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(12));
471 reg_def XMM13n( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(13));
472 reg_def XMM13o( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(14));
473 reg_def XMM13p( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(15));
474
475 reg_def XMM14 ( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg());
476 reg_def XMM14b( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(1));
477 reg_def XMM14c( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(2));
478 reg_def XMM14d( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(3));
479 reg_def XMM14e( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(4));
480 reg_def XMM14f( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(5));
481 reg_def XMM14g( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(6));
482 reg_def XMM14h( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(7));
483 reg_def XMM14i( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(8));
484 reg_def XMM14j( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(9));
485 reg_def XMM14k( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(10));
486 reg_def XMM14l( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(11));
487 reg_def XMM14m( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(12));
488 reg_def XMM14n( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(13));
489 reg_def XMM14o( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(14));
490 reg_def XMM14p( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(15));
491
492 reg_def XMM15 ( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg());
493 reg_def XMM15b( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(1));
494 reg_def XMM15c( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(2));
495 reg_def XMM15d( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(3));
496 reg_def XMM15e( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(4));
497 reg_def XMM15f( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(5));
498 reg_def XMM15g( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(6));
499 reg_def XMM15h( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(7));
500 reg_def XMM15i( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(8));
501 reg_def XMM15j( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(9));
502 reg_def XMM15k( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(10));
503 reg_def XMM15l( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(11));
504 reg_def XMM15m( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(12));
505 reg_def XMM15n( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(13));
506 reg_def XMM15o( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(14));
507 reg_def XMM15p( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(15));
508
509 reg_def XMM16 ( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg());
510 reg_def XMM16b( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(1));
511 reg_def XMM16c( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(2));
512 reg_def XMM16d( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(3));
513 reg_def XMM16e( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(4));
514 reg_def XMM16f( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(5));
515 reg_def XMM16g( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(6));
516 reg_def XMM16h( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(7));
517 reg_def XMM16i( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(8));
518 reg_def XMM16j( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(9));
519 reg_def XMM16k( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(10));
520 reg_def XMM16l( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(11));
521 reg_def XMM16m( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(12));
522 reg_def XMM16n( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(13));
523 reg_def XMM16o( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(14));
524 reg_def XMM16p( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(15));
525
526 reg_def XMM17 ( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg());
527 reg_def XMM17b( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(1));
528 reg_def XMM17c( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(2));
529 reg_def XMM17d( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(3));
530 reg_def XMM17e( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(4));
531 reg_def XMM17f( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(5));
532 reg_def XMM17g( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(6));
533 reg_def XMM17h( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(7));
534 reg_def XMM17i( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(8));
535 reg_def XMM17j( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(9));
536 reg_def XMM17k( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(10));
537 reg_def XMM17l( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(11));
538 reg_def XMM17m( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(12));
539 reg_def XMM17n( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(13));
540 reg_def XMM17o( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(14));
541 reg_def XMM17p( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(15));
542
543 reg_def XMM18 ( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg());
544 reg_def XMM18b( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(1));
545 reg_def XMM18c( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(2));
546 reg_def XMM18d( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(3));
547 reg_def XMM18e( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(4));
548 reg_def XMM18f( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(5));
549 reg_def XMM18g( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(6));
550 reg_def XMM18h( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(7));
551 reg_def XMM18i( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(8));
552 reg_def XMM18j( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(9));
553 reg_def XMM18k( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(10));
554 reg_def XMM18l( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(11));
555 reg_def XMM18m( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(12));
556 reg_def XMM18n( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(13));
557 reg_def XMM18o( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(14));
558 reg_def XMM18p( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(15));
559
560 reg_def XMM19 ( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg());
561 reg_def XMM19b( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(1));
562 reg_def XMM19c( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(2));
563 reg_def XMM19d( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(3));
564 reg_def XMM19e( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(4));
565 reg_def XMM19f( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(5));
566 reg_def XMM19g( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(6));
567 reg_def XMM19h( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(7));
568 reg_def XMM19i( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(8));
569 reg_def XMM19j( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(9));
570 reg_def XMM19k( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(10));
571 reg_def XMM19l( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(11));
572 reg_def XMM19m( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(12));
573 reg_def XMM19n( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(13));
574 reg_def XMM19o( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(14));
575 reg_def XMM19p( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(15));
576
577 reg_def XMM20 ( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg());
578 reg_def XMM20b( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(1));
579 reg_def XMM20c( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(2));
580 reg_def XMM20d( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(3));
581 reg_def XMM20e( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(4));
582 reg_def XMM20f( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(5));
583 reg_def XMM20g( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(6));
584 reg_def XMM20h( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(7));
585 reg_def XMM20i( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(8));
586 reg_def XMM20j( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(9));
587 reg_def XMM20k( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(10));
588 reg_def XMM20l( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(11));
589 reg_def XMM20m( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(12));
590 reg_def XMM20n( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(13));
591 reg_def XMM20o( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(14));
592 reg_def XMM20p( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(15));
593
594 reg_def XMM21 ( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg());
595 reg_def XMM21b( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(1));
596 reg_def XMM21c( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(2));
597 reg_def XMM21d( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(3));
598 reg_def XMM21e( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(4));
599 reg_def XMM21f( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(5));
600 reg_def XMM21g( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(6));
601 reg_def XMM21h( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(7));
602 reg_def XMM21i( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(8));
603 reg_def XMM21j( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(9));
604 reg_def XMM21k( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(10));
605 reg_def XMM21l( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(11));
606 reg_def XMM21m( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(12));
607 reg_def XMM21n( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(13));
608 reg_def XMM21o( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(14));
609 reg_def XMM21p( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(15));
610
611 reg_def XMM22 ( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg());
612 reg_def XMM22b( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(1));
613 reg_def XMM22c( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(2));
614 reg_def XMM22d( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(3));
615 reg_def XMM22e( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(4));
616 reg_def XMM22f( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(5));
617 reg_def XMM22g( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(6));
618 reg_def XMM22h( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(7));
619 reg_def XMM22i( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(8));
620 reg_def XMM22j( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(9));
621 reg_def XMM22k( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(10));
622 reg_def XMM22l( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(11));
623 reg_def XMM22m( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(12));
624 reg_def XMM22n( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(13));
625 reg_def XMM22o( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(14));
626 reg_def XMM22p( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(15));
627
628 reg_def XMM23 ( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg());
629 reg_def XMM23b( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(1));
630 reg_def XMM23c( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(2));
631 reg_def XMM23d( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(3));
632 reg_def XMM23e( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(4));
633 reg_def XMM23f( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(5));
634 reg_def XMM23g( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(6));
635 reg_def XMM23h( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(7));
636 reg_def XMM23i( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(8));
637 reg_def XMM23j( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(9));
638 reg_def XMM23k( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(10));
639 reg_def XMM23l( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(11));
640 reg_def XMM23m( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(12));
641 reg_def XMM23n( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(13));
642 reg_def XMM23o( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(14));
643 reg_def XMM23p( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(15));
644
645 reg_def XMM24 ( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg());
646 reg_def XMM24b( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(1));
647 reg_def XMM24c( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(2));
648 reg_def XMM24d( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(3));
649 reg_def XMM24e( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(4));
650 reg_def XMM24f( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(5));
651 reg_def XMM24g( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(6));
652 reg_def XMM24h( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(7));
653 reg_def XMM24i( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(8));
654 reg_def XMM24j( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(9));
655 reg_def XMM24k( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(10));
656 reg_def XMM24l( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(11));
657 reg_def XMM24m( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(12));
658 reg_def XMM24n( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(13));
659 reg_def XMM24o( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(14));
660 reg_def XMM24p( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(15));
661
662 reg_def XMM25 ( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg());
663 reg_def XMM25b( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(1));
664 reg_def XMM25c( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(2));
665 reg_def XMM25d( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(3));
666 reg_def XMM25e( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(4));
667 reg_def XMM25f( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(5));
668 reg_def XMM25g( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(6));
669 reg_def XMM25h( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(7));
670 reg_def XMM25i( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(8));
671 reg_def XMM25j( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(9));
672 reg_def XMM25k( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(10));
673 reg_def XMM25l( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(11));
674 reg_def XMM25m( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(12));
675 reg_def XMM25n( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(13));
676 reg_def XMM25o( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(14));
677 reg_def XMM25p( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(15));
678
679 reg_def XMM26 ( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg());
680 reg_def XMM26b( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(1));
681 reg_def XMM26c( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(2));
682 reg_def XMM26d( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(3));
683 reg_def XMM26e( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(4));
684 reg_def XMM26f( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(5));
685 reg_def XMM26g( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(6));
686 reg_def XMM26h( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(7));
687 reg_def XMM26i( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(8));
688 reg_def XMM26j( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(9));
689 reg_def XMM26k( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(10));
690 reg_def XMM26l( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(11));
691 reg_def XMM26m( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(12));
692 reg_def XMM26n( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(13));
693 reg_def XMM26o( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(14));
694 reg_def XMM26p( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(15));
695
696 reg_def XMM27 ( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg());
697 reg_def XMM27b( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(1));
698 reg_def XMM27c( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(2));
699 reg_def XMM27d( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(3));
700 reg_def XMM27e( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(4));
701 reg_def XMM27f( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(5));
702 reg_def XMM27g( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(6));
703 reg_def XMM27h( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(7));
704 reg_def XMM27i( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(8));
705 reg_def XMM27j( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(9));
706 reg_def XMM27k( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(10));
707 reg_def XMM27l( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(11));
708 reg_def XMM27m( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(12));
709 reg_def XMM27n( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(13));
710 reg_def XMM27o( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(14));
711 reg_def XMM27p( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(15));
712
713 reg_def XMM28 ( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg());
714 reg_def XMM28b( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(1));
715 reg_def XMM28c( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(2));
716 reg_def XMM28d( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(3));
717 reg_def XMM28e( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(4));
718 reg_def XMM28f( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(5));
719 reg_def XMM28g( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(6));
720 reg_def XMM28h( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(7));
721 reg_def XMM28i( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(8));
722 reg_def XMM28j( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(9));
723 reg_def XMM28k( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(10));
724 reg_def XMM28l( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(11));
725 reg_def XMM28m( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(12));
726 reg_def XMM28n( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(13));
727 reg_def XMM28o( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(14));
728 reg_def XMM28p( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(15));
729
730 reg_def XMM29 ( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg());
731 reg_def XMM29b( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(1));
732 reg_def XMM29c( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(2));
733 reg_def XMM29d( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(3));
734 reg_def XMM29e( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(4));
735 reg_def XMM29f( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(5));
736 reg_def XMM29g( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(6));
737 reg_def XMM29h( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(7));
738 reg_def XMM29i( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(8));
739 reg_def XMM29j( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(9));
740 reg_def XMM29k( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(10));
741 reg_def XMM29l( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(11));
742 reg_def XMM29m( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(12));
743 reg_def XMM29n( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(13));
744 reg_def XMM29o( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(14));
745 reg_def XMM29p( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(15));
746
747 reg_def XMM30 ( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg());
748 reg_def XMM30b( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(1));
749 reg_def XMM30c( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(2));
750 reg_def XMM30d( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(3));
751 reg_def XMM30e( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(4));
752 reg_def XMM30f( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(5));
753 reg_def XMM30g( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(6));
754 reg_def XMM30h( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(7));
755 reg_def XMM30i( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(8));
756 reg_def XMM30j( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(9));
757 reg_def XMM30k( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(10));
758 reg_def XMM30l( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(11));
759 reg_def XMM30m( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(12));
760 reg_def XMM30n( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(13));
761 reg_def XMM30o( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(14));
762 reg_def XMM30p( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(15));
763
764 reg_def XMM31 ( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg());
765 reg_def XMM31b( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(1));
766 reg_def XMM31c( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(2));
767 reg_def XMM31d( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(3));
768 reg_def XMM31e( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(4));
769 reg_def XMM31f( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(5));
770 reg_def XMM31g( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(6));
771 reg_def XMM31h( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(7));
772 reg_def XMM31i( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(8));
773 reg_def XMM31j( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(9));
774 reg_def XMM31k( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(10));
775 reg_def XMM31l( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(11));
776 reg_def XMM31m( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(12));
777 reg_def XMM31n( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(13));
778 reg_def XMM31o( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(14));
779 reg_def XMM31p( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(15));
780
781 reg_def RFLAGS(SOC, SOC, 0, 16, VMRegImpl::Bad());
782
783 // AVX3 Mask Registers.
784 reg_def K1 (SOC, SOC, Op_RegI, 1, k1->as_VMReg());
785 reg_def K1_H (SOC, SOC, Op_RegI, 1, k1->as_VMReg()->next());
786
787 reg_def K2 (SOC, SOC, Op_RegI, 2, k2->as_VMReg());
788 reg_def K2_H (SOC, SOC, Op_RegI, 2, k2->as_VMReg()->next());
789
790 reg_def K3 (SOC, SOC, Op_RegI, 3, k3->as_VMReg());
791 reg_def K3_H (SOC, SOC, Op_RegI, 3, k3->as_VMReg()->next());
792
793 reg_def K4 (SOC, SOC, Op_RegI, 4, k4->as_VMReg());
794 reg_def K4_H (SOC, SOC, Op_RegI, 4, k4->as_VMReg()->next());
795
796 reg_def K5 (SOC, SOC, Op_RegI, 5, k5->as_VMReg());
797 reg_def K5_H (SOC, SOC, Op_RegI, 5, k5->as_VMReg()->next());
798
799 reg_def K6 (SOC, SOC, Op_RegI, 6, k6->as_VMReg());
800 reg_def K6_H (SOC, SOC, Op_RegI, 6, k6->as_VMReg()->next());
801
802 reg_def K7 (SOC, SOC, Op_RegI, 7, k7->as_VMReg());
803 reg_def K7_H (SOC, SOC, Op_RegI, 7, k7->as_VMReg()->next());
804
805
806 //----------Architecture Description Register Classes--------------------------
807 // Several register classes are automatically defined based upon information in
808 // this architecture description.
809 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
810 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
811 //
812
813 // Empty register class.
814 reg_class no_reg();
815
816 // Class for all pointer/long registers including APX extended GPRs.
817 reg_class all_reg(RAX, RAX_H,
818 RDX, RDX_H,
819 RBP, RBP_H,
820 RDI, RDI_H,
821 RSI, RSI_H,
822 RCX, RCX_H,
823 RBX, RBX_H,
824 RSP, RSP_H,
825 R8, R8_H,
826 R9, R9_H,
827 R10, R10_H,
828 R11, R11_H,
829 R12, R12_H,
830 R13, R13_H,
831 R14, R14_H,
832 R15, R15_H,
833 R16, R16_H,
834 R17, R17_H,
835 R18, R18_H,
836 R19, R19_H,
837 R20, R20_H,
838 R21, R21_H,
839 R22, R22_H,
840 R23, R23_H,
841 R24, R24_H,
842 R25, R25_H,
843 R26, R26_H,
844 R27, R27_H,
845 R28, R28_H,
846 R29, R29_H,
847 R30, R30_H,
848 R31, R31_H);
849
850 // Class for all int registers including APX extended GPRs.
851 reg_class all_int_reg(RAX
852 RDX,
853 RBP,
854 RDI,
855 RSI,
856 RCX,
857 RBX,
858 R8,
859 R9,
860 R10,
861 R11,
862 R12,
863 R13,
864 R14,
865 R16,
866 R17,
867 R18,
868 R19,
869 R20,
870 R21,
871 R22,
872 R23,
873 R24,
874 R25,
875 R26,
876 R27,
877 R28,
878 R29,
879 R30,
880 R31);
881
882 // Class for all pointer registers
883 reg_class any_reg %{
884 return _ANY_REG_mask;
885 %}
886
887 // Class for all pointer registers (excluding RSP)
888 reg_class ptr_reg %{
889 return _PTR_REG_mask;
890 %}
891
892 // Class for all pointer registers (excluding RSP and RBP)
893 reg_class ptr_reg_no_rbp %{
894 return _PTR_REG_NO_RBP_mask;
895 %}
896
897 // Class for all pointer registers (excluding RAX and RSP)
898 reg_class ptr_no_rax_reg %{
899 return _PTR_NO_RAX_REG_mask;
900 %}
901
902 // Class for all pointer registers (excluding RAX, RBX, and RSP)
903 reg_class ptr_no_rax_rbx_reg %{
904 return _PTR_NO_RAX_RBX_REG_mask;
905 %}
906
907 // Class for all long registers (excluding RSP)
908 reg_class long_reg %{
909 return _LONG_REG_mask;
910 %}
911
912 // Class for all long registers (excluding RAX, RDX and RSP)
913 reg_class long_no_rax_rdx_reg %{
914 return _LONG_NO_RAX_RDX_REG_mask;
915 %}
916
917 // Class for all long registers (excluding RCX and RSP)
918 reg_class long_no_rcx_reg %{
919 return _LONG_NO_RCX_REG_mask;
920 %}
921
922 // Class for all long registers (excluding RBP and R13)
923 reg_class long_no_rbp_r13_reg %{
924 return _LONG_NO_RBP_R13_REG_mask;
925 %}
926
927 // Class for all int registers (excluding RSP)
928 reg_class int_reg %{
929 return _INT_REG_mask;
930 %}
931
932 // Class for all int registers (excluding RAX, RDX, and RSP)
933 reg_class int_no_rax_rdx_reg %{
934 return _INT_NO_RAX_RDX_REG_mask;
935 %}
936
937 // Class for all int registers (excluding RCX and RSP)
938 reg_class int_no_rcx_reg %{
939 return _INT_NO_RCX_REG_mask;
940 %}
941
942 // Class for all int registers (excluding RBP and R13)
943 reg_class int_no_rbp_r13_reg %{
944 return _INT_NO_RBP_R13_REG_mask;
945 %}
946
947 // Singleton class for RAX pointer register
948 reg_class ptr_rax_reg(RAX, RAX_H);
949
950 // Singleton class for RBX pointer register
951 reg_class ptr_rbx_reg(RBX, RBX_H);
952
953 // Singleton class for RSI pointer register
954 reg_class ptr_rsi_reg(RSI, RSI_H);
955
956 // Singleton class for RBP pointer register
957 reg_class ptr_rbp_reg(RBP, RBP_H);
958
959 // Singleton class for RDI pointer register
960 reg_class ptr_rdi_reg(RDI, RDI_H);
961
962 // Singleton class for stack pointer
963 reg_class ptr_rsp_reg(RSP, RSP_H);
964
965 // Singleton class for TLS pointer
966 reg_class ptr_r15_reg(R15, R15_H);
967
968 // Singleton class for RAX long register
969 reg_class long_rax_reg(RAX, RAX_H);
970
971 // Singleton class for RCX long register
972 reg_class long_rcx_reg(RCX, RCX_H);
973
974 // Singleton class for RDX long register
975 reg_class long_rdx_reg(RDX, RDX_H);
976
977 // Singleton class for R11 long register
978 reg_class long_r11_reg(R11, R11_H);
979
980 // Singleton class for RAX int register
981 reg_class int_rax_reg(RAX);
982
983 // Singleton class for RBX int register
984 reg_class int_rbx_reg(RBX);
985
986 // Singleton class for RCX int register
987 reg_class int_rcx_reg(RCX);
988
989 // Singleton class for RDX int register
990 reg_class int_rdx_reg(RDX);
991
992 // Singleton class for RDI int register
993 reg_class int_rdi_reg(RDI);
994
995 // Singleton class for instruction pointer
996 // reg_class ip_reg(RIP);
997
998 alloc_class chunk1(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p,
999 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p,
1000 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p,
1001 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p,
1002 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p,
1003 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p,
1004 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p,
1005 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p,
1006 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p,
1007 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p,
1008 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
1009 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
1010 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
1011 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
1012 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
1013 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p,
1014 XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p,
1015 XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p,
1016 XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p,
1017 XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p,
1018 XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p,
1019 XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p,
1020 XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p,
1021 XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p,
1022 XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p,
1023 XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p,
1024 XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p,
1025 XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p,
1026 XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p,
1027 XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p,
1028 XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p,
1029 XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p);
1030
1031 alloc_class chunk2(K7, K7_H,
1032 K6, K6_H,
1033 K5, K5_H,
1034 K4, K4_H,
1035 K3, K3_H,
1036 K2, K2_H,
1037 K1, K1_H);
1038
1039 reg_class vectmask_reg(K1, K1_H,
1040 K2, K2_H,
1041 K3, K3_H,
1042 K4, K4_H,
1043 K5, K5_H,
1044 K6, K6_H,
1045 K7, K7_H);
1046
1047 reg_class vectmask_reg_K1(K1, K1_H);
1048 reg_class vectmask_reg_K2(K2, K2_H);
1049 reg_class vectmask_reg_K3(K3, K3_H);
1050 reg_class vectmask_reg_K4(K4, K4_H);
1051 reg_class vectmask_reg_K5(K5, K5_H);
1052 reg_class vectmask_reg_K6(K6, K6_H);
1053 reg_class vectmask_reg_K7(K7, K7_H);
1054
1055 // flags allocation class should be last.
1056 alloc_class chunk3(RFLAGS);
1057
1058 // Singleton class for condition codes
1059 reg_class int_flags(RFLAGS);
1060
1061 // Class for pre evex float registers
1062 reg_class float_reg_legacy(XMM0,
1063 XMM1,
1064 XMM2,
1065 XMM3,
1066 XMM4,
1067 XMM5,
1068 XMM6,
1069 XMM7,
1070 XMM8,
1071 XMM9,
1072 XMM10,
1073 XMM11,
1074 XMM12,
1075 XMM13,
1076 XMM14,
1077 XMM15);
1078
1079 // Class for evex float registers
1080 reg_class float_reg_evex(XMM0,
1081 XMM1,
1082 XMM2,
1083 XMM3,
1084 XMM4,
1085 XMM5,
1086 XMM6,
1087 XMM7,
1088 XMM8,
1089 XMM9,
1090 XMM10,
1091 XMM11,
1092 XMM12,
1093 XMM13,
1094 XMM14,
1095 XMM15,
1096 XMM16,
1097 XMM17,
1098 XMM18,
1099 XMM19,
1100 XMM20,
1101 XMM21,
1102 XMM22,
1103 XMM23,
1104 XMM24,
1105 XMM25,
1106 XMM26,
1107 XMM27,
1108 XMM28,
1109 XMM29,
1110 XMM30,
1111 XMM31);
1112
1113 reg_class_dynamic float_reg(float_reg_evex, float_reg_legacy, %{ VM_Version::supports_evex() %} );
1114 reg_class_dynamic float_reg_vl(float_reg_evex, float_reg_legacy, %{ VM_Version::supports_evex() && VM_Version::supports_avx512vl() %} );
1115
1116 // Class for pre evex double registers
1117 reg_class double_reg_legacy(XMM0, XMM0b,
1118 XMM1, XMM1b,
1119 XMM2, XMM2b,
1120 XMM3, XMM3b,
1121 XMM4, XMM4b,
1122 XMM5, XMM5b,
1123 XMM6, XMM6b,
1124 XMM7, XMM7b,
1125 XMM8, XMM8b,
1126 XMM9, XMM9b,
1127 XMM10, XMM10b,
1128 XMM11, XMM11b,
1129 XMM12, XMM12b,
1130 XMM13, XMM13b,
1131 XMM14, XMM14b,
1132 XMM15, XMM15b);
1133
1134 // Class for evex double registers
1135 reg_class double_reg_evex(XMM0, XMM0b,
1136 XMM1, XMM1b,
1137 XMM2, XMM2b,
1138 XMM3, XMM3b,
1139 XMM4, XMM4b,
1140 XMM5, XMM5b,
1141 XMM6, XMM6b,
1142 XMM7, XMM7b,
1143 XMM8, XMM8b,
1144 XMM9, XMM9b,
1145 XMM10, XMM10b,
1146 XMM11, XMM11b,
1147 XMM12, XMM12b,
1148 XMM13, XMM13b,
1149 XMM14, XMM14b,
1150 XMM15, XMM15b,
1151 XMM16, XMM16b,
1152 XMM17, XMM17b,
1153 XMM18, XMM18b,
1154 XMM19, XMM19b,
1155 XMM20, XMM20b,
1156 XMM21, XMM21b,
1157 XMM22, XMM22b,
1158 XMM23, XMM23b,
1159 XMM24, XMM24b,
1160 XMM25, XMM25b,
1161 XMM26, XMM26b,
1162 XMM27, XMM27b,
1163 XMM28, XMM28b,
1164 XMM29, XMM29b,
1165 XMM30, XMM30b,
1166 XMM31, XMM31b);
1167
1168 reg_class_dynamic double_reg(double_reg_evex, double_reg_legacy, %{ VM_Version::supports_evex() %} );
1169 reg_class_dynamic double_reg_vl(double_reg_evex, double_reg_legacy, %{ VM_Version::supports_evex() && VM_Version::supports_avx512vl() %} );
1170
1171 // Class for pre evex 32bit vector registers
1172 reg_class vectors_reg_legacy(XMM0,
1173 XMM1,
1174 XMM2,
1175 XMM3,
1176 XMM4,
1177 XMM5,
1178 XMM6,
1179 XMM7,
1180 XMM8,
1181 XMM9,
1182 XMM10,
1183 XMM11,
1184 XMM12,
1185 XMM13,
1186 XMM14,
1187 XMM15);
1188
1189 // Class for evex 32bit vector registers
1190 reg_class vectors_reg_evex(XMM0,
1191 XMM1,
1192 XMM2,
1193 XMM3,
1194 XMM4,
1195 XMM5,
1196 XMM6,
1197 XMM7,
1198 XMM8,
1199 XMM9,
1200 XMM10,
1201 XMM11,
1202 XMM12,
1203 XMM13,
1204 XMM14,
1205 XMM15,
1206 XMM16,
1207 XMM17,
1208 XMM18,
1209 XMM19,
1210 XMM20,
1211 XMM21,
1212 XMM22,
1213 XMM23,
1214 XMM24,
1215 XMM25,
1216 XMM26,
1217 XMM27,
1218 XMM28,
1219 XMM29,
1220 XMM30,
1221 XMM31);
1222
1223 reg_class_dynamic vectors_reg(vectors_reg_evex, vectors_reg_legacy, %{ VM_Version::supports_evex() %} );
1224 reg_class_dynamic vectors_reg_vlbwdq(vectors_reg_evex, vectors_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1225
1226 // Class for all 64bit vector registers
1227 reg_class vectord_reg_legacy(XMM0, XMM0b,
1228 XMM1, XMM1b,
1229 XMM2, XMM2b,
1230 XMM3, XMM3b,
1231 XMM4, XMM4b,
1232 XMM5, XMM5b,
1233 XMM6, XMM6b,
1234 XMM7, XMM7b,
1235 XMM8, XMM8b,
1236 XMM9, XMM9b,
1237 XMM10, XMM10b,
1238 XMM11, XMM11b,
1239 XMM12, XMM12b,
1240 XMM13, XMM13b,
1241 XMM14, XMM14b,
1242 XMM15, XMM15b);
1243
1244 // Class for all 64bit vector registers
1245 reg_class vectord_reg_evex(XMM0, XMM0b,
1246 XMM1, XMM1b,
1247 XMM2, XMM2b,
1248 XMM3, XMM3b,
1249 XMM4, XMM4b,
1250 XMM5, XMM5b,
1251 XMM6, XMM6b,
1252 XMM7, XMM7b,
1253 XMM8, XMM8b,
1254 XMM9, XMM9b,
1255 XMM10, XMM10b,
1256 XMM11, XMM11b,
1257 XMM12, XMM12b,
1258 XMM13, XMM13b,
1259 XMM14, XMM14b,
1260 XMM15, XMM15b,
1261 XMM16, XMM16b,
1262 XMM17, XMM17b,
1263 XMM18, XMM18b,
1264 XMM19, XMM19b,
1265 XMM20, XMM20b,
1266 XMM21, XMM21b,
1267 XMM22, XMM22b,
1268 XMM23, XMM23b,
1269 XMM24, XMM24b,
1270 XMM25, XMM25b,
1271 XMM26, XMM26b,
1272 XMM27, XMM27b,
1273 XMM28, XMM28b,
1274 XMM29, XMM29b,
1275 XMM30, XMM30b,
1276 XMM31, XMM31b);
1277
1278 reg_class_dynamic vectord_reg(vectord_reg_evex, vectord_reg_legacy, %{ VM_Version::supports_evex() %} );
1279 reg_class_dynamic vectord_reg_vlbwdq(vectord_reg_evex, vectord_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1280
1281 // Class for all 128bit vector registers
1282 reg_class vectorx_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d,
1283 XMM1, XMM1b, XMM1c, XMM1d,
1284 XMM2, XMM2b, XMM2c, XMM2d,
1285 XMM3, XMM3b, XMM3c, XMM3d,
1286 XMM4, XMM4b, XMM4c, XMM4d,
1287 XMM5, XMM5b, XMM5c, XMM5d,
1288 XMM6, XMM6b, XMM6c, XMM6d,
1289 XMM7, XMM7b, XMM7c, XMM7d,
1290 XMM8, XMM8b, XMM8c, XMM8d,
1291 XMM9, XMM9b, XMM9c, XMM9d,
1292 XMM10, XMM10b, XMM10c, XMM10d,
1293 XMM11, XMM11b, XMM11c, XMM11d,
1294 XMM12, XMM12b, XMM12c, XMM12d,
1295 XMM13, XMM13b, XMM13c, XMM13d,
1296 XMM14, XMM14b, XMM14c, XMM14d,
1297 XMM15, XMM15b, XMM15c, XMM15d);
1298
1299 // Class for all 128bit vector registers
1300 reg_class vectorx_reg_evex(XMM0, XMM0b, XMM0c, XMM0d,
1301 XMM1, XMM1b, XMM1c, XMM1d,
1302 XMM2, XMM2b, XMM2c, XMM2d,
1303 XMM3, XMM3b, XMM3c, XMM3d,
1304 XMM4, XMM4b, XMM4c, XMM4d,
1305 XMM5, XMM5b, XMM5c, XMM5d,
1306 XMM6, XMM6b, XMM6c, XMM6d,
1307 XMM7, XMM7b, XMM7c, XMM7d,
1308 XMM8, XMM8b, XMM8c, XMM8d,
1309 XMM9, XMM9b, XMM9c, XMM9d,
1310 XMM10, XMM10b, XMM10c, XMM10d,
1311 XMM11, XMM11b, XMM11c, XMM11d,
1312 XMM12, XMM12b, XMM12c, XMM12d,
1313 XMM13, XMM13b, XMM13c, XMM13d,
1314 XMM14, XMM14b, XMM14c, XMM14d,
1315 XMM15, XMM15b, XMM15c, XMM15d,
1316 XMM16, XMM16b, XMM16c, XMM16d,
1317 XMM17, XMM17b, XMM17c, XMM17d,
1318 XMM18, XMM18b, XMM18c, XMM18d,
1319 XMM19, XMM19b, XMM19c, XMM19d,
1320 XMM20, XMM20b, XMM20c, XMM20d,
1321 XMM21, XMM21b, XMM21c, XMM21d,
1322 XMM22, XMM22b, XMM22c, XMM22d,
1323 XMM23, XMM23b, XMM23c, XMM23d,
1324 XMM24, XMM24b, XMM24c, XMM24d,
1325 XMM25, XMM25b, XMM25c, XMM25d,
1326 XMM26, XMM26b, XMM26c, XMM26d,
1327 XMM27, XMM27b, XMM27c, XMM27d,
1328 XMM28, XMM28b, XMM28c, XMM28d,
1329 XMM29, XMM29b, XMM29c, XMM29d,
1330 XMM30, XMM30b, XMM30c, XMM30d,
1331 XMM31, XMM31b, XMM31c, XMM31d);
1332
1333 reg_class_dynamic vectorx_reg(vectorx_reg_evex, vectorx_reg_legacy, %{ VM_Version::supports_evex() %} );
1334 reg_class_dynamic vectorx_reg_vlbwdq(vectorx_reg_evex, vectorx_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1335
1336 // Class for all 256bit vector registers
1337 reg_class vectory_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h,
1338 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h,
1339 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h,
1340 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h,
1341 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h,
1342 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h,
1343 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h,
1344 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h,
1345 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h,
1346 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h,
1347 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h,
1348 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h,
1349 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h,
1350 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h,
1351 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h,
1352 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h);
1353
1354 // Class for all 256bit vector registers
1355 reg_class vectory_reg_evex(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h,
1356 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h,
1357 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h,
1358 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h,
1359 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h,
1360 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h,
1361 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h,
1362 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h,
1363 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h,
1364 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h,
1365 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h,
1366 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h,
1367 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h,
1368 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h,
1369 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h,
1370 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h,
1371 XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h,
1372 XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h,
1373 XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h,
1374 XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h,
1375 XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h,
1376 XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h,
1377 XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h,
1378 XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h,
1379 XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h,
1380 XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h,
1381 XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h,
1382 XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h,
1383 XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h,
1384 XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h,
1385 XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h,
1386 XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h);
1387
1388 reg_class_dynamic vectory_reg(vectory_reg_evex, vectory_reg_legacy, %{ VM_Version::supports_evex() %} );
1389 reg_class_dynamic vectory_reg_vlbwdq(vectory_reg_evex, vectory_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1390
1391 // Class for all 512bit vector registers
1392 reg_class vectorz_reg_evex(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p,
1393 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p,
1394 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p,
1395 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p,
1396 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p,
1397 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p,
1398 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p,
1399 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p,
1400 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p,
1401 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p,
1402 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
1403 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
1404 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
1405 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
1406 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
1407 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p,
1408 XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p,
1409 XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p,
1410 XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p,
1411 XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p,
1412 XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p,
1413 XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p,
1414 XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p,
1415 XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p,
1416 XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p,
1417 XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p,
1418 XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p,
1419 XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p,
1420 XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p,
1421 XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p,
1422 XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p,
1423 XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p);
1424
1425 // Class for restricted 512bit vector registers
1426 reg_class vectorz_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p,
1427 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p,
1428 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p,
1429 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p,
1430 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p,
1431 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p,
1432 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p,
1433 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p,
1434 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p,
1435 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p,
1436 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
1437 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
1438 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
1439 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
1440 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
1441 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p);
1442
1443 reg_class_dynamic vectorz_reg (vectorz_reg_evex, vectorz_reg_legacy, %{ VM_Version::supports_evex() %} );
1444 reg_class_dynamic vectorz_reg_vl(vectorz_reg_evex, vectorz_reg_legacy, %{ VM_Version::supports_evex() && VM_Version::supports_avx512vl() %} );
1445
1446 reg_class xmm0_reg(XMM0, XMM0b, XMM0c, XMM0d);
1447
1448 %}
1449
1450
1451 //----------SOURCE BLOCK-------------------------------------------------------
1452 // This is a block of C++ code which provides values, functions, and
1453 // definitions necessary in the rest of the architecture description
1454
1455 source_hpp %{
1456
1457 #include "peephole_x86_64.hpp"
1458
1459 bool castLL_is_imm32(const Node* n);
1460
1461 %}
1462
1463 source %{
1464
1465 bool castLL_is_imm32(const Node* n) {
1466 assert(n->is_CastLL(), "must be a CastLL");
1467 const TypeLong* t = n->bottom_type()->is_long();
1468 return (t->_lo == min_jlong || Assembler::is_simm32(t->_lo)) && (t->_hi == max_jlong || Assembler::is_simm32(t->_hi));
1469 }
1470
1471 %}
1472
1473 // Register masks
1474 source_hpp %{
1475
1476 extern RegMask _ANY_REG_mask;
1477 extern RegMask _PTR_REG_mask;
1478 extern RegMask _PTR_REG_NO_RBP_mask;
1479 extern RegMask _PTR_NO_RAX_REG_mask;
1480 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
1481 extern RegMask _LONG_REG_mask;
1482 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
1483 extern RegMask _LONG_NO_RCX_REG_mask;
1484 extern RegMask _LONG_NO_RBP_R13_REG_mask;
1485 extern RegMask _INT_REG_mask;
1486 extern RegMask _INT_NO_RAX_RDX_REG_mask;
1487 extern RegMask _INT_NO_RCX_REG_mask;
1488 extern RegMask _INT_NO_RBP_R13_REG_mask;
1489 extern RegMask _FLOAT_REG_mask;
1490
1491 extern RegMask _STACK_OR_PTR_REG_mask;
1492 extern RegMask _STACK_OR_LONG_REG_mask;
1493 extern RegMask _STACK_OR_INT_REG_mask;
1494
1495 inline const RegMask& STACK_OR_PTR_REG_mask() { return _STACK_OR_PTR_REG_mask; }
1496 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
1497 inline const RegMask& STACK_OR_INT_REG_mask() { return _STACK_OR_INT_REG_mask; }
1498
1499 %}
1500
1501 source %{
1502 #define RELOC_IMM64 Assembler::imm_operand
1503 #define RELOC_DISP32 Assembler::disp32_operand
1504
1505 #define __ masm->
1506
1507 RegMask _ANY_REG_mask;
1508 RegMask _PTR_REG_mask;
1509 RegMask _PTR_REG_NO_RBP_mask;
1510 RegMask _PTR_NO_RAX_REG_mask;
1511 RegMask _PTR_NO_RAX_RBX_REG_mask;
1512 RegMask _LONG_REG_mask;
1513 RegMask _LONG_NO_RAX_RDX_REG_mask;
1514 RegMask _LONG_NO_RCX_REG_mask;
1515 RegMask _LONG_NO_RBP_R13_REG_mask;
1516 RegMask _INT_REG_mask;
1517 RegMask _INT_NO_RAX_RDX_REG_mask;
1518 RegMask _INT_NO_RCX_REG_mask;
1519 RegMask _INT_NO_RBP_R13_REG_mask;
1520 RegMask _FLOAT_REG_mask;
1521 RegMask _STACK_OR_PTR_REG_mask;
1522 RegMask _STACK_OR_LONG_REG_mask;
1523 RegMask _STACK_OR_INT_REG_mask;
1524
1525 static bool need_r12_heapbase() {
1526 return UseCompressedOops;
1527 }
1528
1529 void reg_mask_init() {
1530 constexpr Register egprs[] = {r16, r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31};
1531
1532 // _ALL_REG_mask is generated by adlc from the all_reg register class below.
1533 // We derive a number of subsets from it.
1534 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1535
1536 if (PreserveFramePointer) {
1537 _ANY_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1538 _ANY_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
1539 }
1540 if (need_r12_heapbase()) {
1541 _ANY_REG_mask.remove(OptoReg::as_OptoReg(r12->as_VMReg()));
1542 _ANY_REG_mask.remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
1543 }
1544
1545 _PTR_REG_mask.assignFrom(_ANY_REG_mask);
1546 _PTR_REG_mask.remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
1547 _PTR_REG_mask.remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
1548 _PTR_REG_mask.remove(OptoReg::as_OptoReg(r15->as_VMReg()));
1549 _PTR_REG_mask.remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
1550 if (!UseAPX) {
1551 for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
1552 _PTR_REG_mask.remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
1553 _PTR_REG_mask.remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()->next()));
1554 }
1555 }
1556
1557 _STACK_OR_PTR_REG_mask.assignFrom(_PTR_REG_mask);
1558 _STACK_OR_PTR_REG_mask.or_with(STACK_OR_STACK_SLOTS_mask());
1559
1560 _PTR_REG_NO_RBP_mask.assignFrom(_PTR_REG_mask);
1561 _PTR_REG_NO_RBP_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1562 _PTR_REG_NO_RBP_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
1563
1564 _PTR_NO_RAX_REG_mask.assignFrom(_PTR_REG_mask);
1565 _PTR_NO_RAX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()));
1566 _PTR_NO_RAX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
1567
1568 _PTR_NO_RAX_RBX_REG_mask.assignFrom(_PTR_NO_RAX_REG_mask);
1569 _PTR_NO_RAX_RBX_REG_mask.remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
1570 _PTR_NO_RAX_RBX_REG_mask.remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
1571
1572
1573 _LONG_REG_mask.assignFrom(_PTR_REG_mask);
1574 _STACK_OR_LONG_REG_mask.assignFrom(_LONG_REG_mask);
1575 _STACK_OR_LONG_REG_mask.or_with(STACK_OR_STACK_SLOTS_mask());
1576
1577 _LONG_NO_RAX_RDX_REG_mask.assignFrom(_LONG_REG_mask);
1578 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()));
1579 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
1580 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
1581 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
1582
1583 _LONG_NO_RCX_REG_mask.assignFrom(_LONG_REG_mask);
1584 _LONG_NO_RCX_REG_mask.remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
1585 _LONG_NO_RCX_REG_mask.remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
1586
1587 _LONG_NO_RBP_R13_REG_mask.assignFrom(_LONG_REG_mask);
1588 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1589 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
1590 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(r13->as_VMReg()));
1591 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
1592
1593 _INT_REG_mask.assignFrom(_ALL_INT_REG_mask);
1594 if (!UseAPX) {
1595 for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
1596 _INT_REG_mask.remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
1597 }
1598 }
1599
1600 if (PreserveFramePointer) {
1601 _INT_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1602 }
1603 if (need_r12_heapbase()) {
1604 _INT_REG_mask.remove(OptoReg::as_OptoReg(r12->as_VMReg()));
1605 }
1606
1607 _STACK_OR_INT_REG_mask.assignFrom(_INT_REG_mask);
1608 _STACK_OR_INT_REG_mask.or_with(STACK_OR_STACK_SLOTS_mask());
1609
1610 _INT_NO_RAX_RDX_REG_mask.assignFrom(_INT_REG_mask);
1611 _INT_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()));
1612 _INT_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
1613
1614 _INT_NO_RCX_REG_mask.assignFrom(_INT_REG_mask);
1615 _INT_NO_RCX_REG_mask.remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
1616
1617 _INT_NO_RBP_R13_REG_mask.assignFrom(_INT_REG_mask);
1618 _INT_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1619 _INT_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(r13->as_VMReg()));
1620
1621 // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
1622 // from the float_reg_legacy/float_reg_evex register class.
1623 _FLOAT_REG_mask.assignFrom(VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask);
1624 }
1625
1626 static bool generate_vzeroupper(Compile* C) {
1627 return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false; // Generate vzeroupper
1628 }
1629
1630 static int clear_avx_size() {
1631 return generate_vzeroupper(Compile::current()) ? 3: 0; // vzeroupper
1632 }
1633
1634 // !!!!! Special hack to get all types of calls to specify the byte offset
1635 // from the start of the call to the point where the return address
1636 // will point.
1637 int MachCallStaticJavaNode::ret_addr_offset()
1638 {
1639 int offset = 5; // 5 bytes from start of call to where return address points
1640 offset += clear_avx_size();
1641 return offset;
1642 }
1643
1644 int MachCallDynamicJavaNode::ret_addr_offset()
1645 {
1646 int offset = 15; // 15 bytes from start of call to where return address points
1647 offset += clear_avx_size();
1648 return offset;
1649 }
1650
1651 int MachCallRuntimeNode::ret_addr_offset() {
1652 int offset = 13; // movq r10,#addr; callq (r10)
1653 if (this->ideal_Opcode() != Op_CallLeafVector) {
1654 offset += clear_avx_size();
1655 }
1656 return offset;
1657 }
1658 //
1659 // Compute padding required for nodes which need alignment
1660 //
1661
1662 // The address of the call instruction needs to be 4-byte aligned to
1663 // ensure that it does not span a cache line so that it can be patched.
1664 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
1665 {
1666 current_offset += clear_avx_size(); // skip vzeroupper
1667 current_offset += 1; // skip call opcode byte
1668 return align_up(current_offset, alignment_required()) - current_offset;
1669 }
1670
1671 // The address of the call instruction needs to be 4-byte aligned to
1672 // ensure that it does not span a cache line so that it can be patched.
1673 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
1674 {
1675 current_offset += clear_avx_size(); // skip vzeroupper
1676 current_offset += 11; // skip movq instruction + call opcode byte
1677 return align_up(current_offset, alignment_required()) - current_offset;
1678 }
1679
1680 // This could be in MacroAssembler but it's fairly C2 specific
1681 static void emit_cmpfp_fixup(MacroAssembler* masm) {
1682 Label exit;
1683 __ jccb(Assembler::noParity, exit);
1684 __ pushf();
1685 //
1686 // comiss/ucomiss instructions set ZF,PF,CF flags and
1687 // zero OF,AF,SF for NaN values.
1688 // Fixup flags by zeroing ZF,PF so that compare of NaN
1689 // values returns 'less than' result (CF is set).
1690 // Leave the rest of flags unchanged.
1691 //
1692 // 7 6 5 4 3 2 1 0
1693 // |S|Z|r|A|r|P|r|C| (r - reserved bit)
1694 // 0 0 1 0 1 0 1 1 (0x2B)
1695 //
1696 __ andq(Address(rsp, 0), 0xffffff2b);
1697 __ popf();
1698 __ bind(exit);
1699 }
1700
1701 static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
1702 // If any floating point comparison instruction is used, unordered case always triggers jump
1703 // for below condition, CF=1 is true when at least one input is NaN
1704 Label done;
1705 __ movl(dst, -1);
1706 __ jcc(Assembler::below, done);
1707 __ setcc(Assembler::notEqual, dst);
1708 __ bind(done);
1709 }
1710
1711 enum FP_PREC {
1712 fp_prec_hlf,
1713 fp_prec_flt,
1714 fp_prec_dbl
1715 };
1716
1717 static inline void emit_fp_ucom(MacroAssembler* masm, enum FP_PREC pt,
1718 XMMRegister p, XMMRegister q) {
1719 if (pt == fp_prec_hlf) {
1720 __ evucomish(p, q);
1721 } else if (pt == fp_prec_flt) {
1722 __ ucomiss(p, q);
1723 } else {
1724 __ ucomisd(p, q);
1725 }
1726 }
1727
1728 static inline void movfp(MacroAssembler* masm, enum FP_PREC pt,
1729 XMMRegister dst, XMMRegister src, Register scratch) {
1730 if (pt == fp_prec_hlf) {
1731 __ movhlf(dst, src, scratch);
1732 } else if (pt == fp_prec_flt) {
1733 __ movflt(dst, src);
1734 } else {
1735 __ movdbl(dst, src);
1736 }
1737 }
1738
1739 // Math.min() # Math.max()
1740 // -----------------------------
1741 // (v)ucomis[h/s/d] #
1742 // ja -> b # a
1743 // jp -> NaN # NaN
1744 // jb -> a # b
1745 // je -> a | b # a & b
1746 static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
1747 XMMRegister a, XMMRegister b, Register rt,
1748 bool min, enum FP_PREC pt) {
1749 Label nan, zero, below, above, done;
1750
1751 emit_fp_ucom(masm, pt, a, b);
1752
1753 if (dst->encoding() != (min ? b : a)->encoding()) {
1754 __ jccb(Assembler::above, above); // CF=0 & ZF=0
1755 } else {
1756 __ jccb(Assembler::above, done);
1757 }
1758 __ jccb(Assembler::parity, nan); // PF=1
1759 __ jccb(Assembler::below, below); // CF=1
1760
1761 // equal
1762 // Using bitwise operations is a low cost way to compute the correct result
1763 // for zero and non-zero inputs in this scenario except for NaN, which is
1764 // handled separately. The mantissa and exponent are valid with either
1765 // bitwise operation. For zero inputs, the sign bit is chosen according to
1766 // whether a minimum or maximum value is required.
1767 if (min) {
1768 // Negative sign preserved when available (e.g., min(+0, -0) -> -0)
1769 __ vpor(dst, a, b, Assembler::AVX_128bit);
1770 } else {
1771 // Positive sign preserved when available (e.g., max(+0, -0) -> +0)
1772 __ vpand(dst, a, b, Assembler::AVX_128bit);
1773 }
1774 __ jmp(done);
1775
1776 __ bind(above);
1777 movfp(masm, pt, dst, min ? b : a, rt);
1778 __ jmp(done);
1779
1780 __ bind(nan);
1781 if (pt == fp_prec_hlf) {
1782 __ movl(rt, 0x00007e00); // Float16.NaN
1783 __ evmovw(dst, rt);
1784 } else if (pt == fp_prec_flt) {
1785 __ movl(rt, 0x7fc00000); // Float.NaN
1786 __ movdl(dst, rt);
1787 } else {
1788 __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
1789 __ movdq(dst, rt);
1790 }
1791 __ jmp(done);
1792
1793 __ bind(below);
1794 movfp(masm, pt, dst, min ? a : b, rt);
1795
1796 __ bind(done);
1797 }
1798
1799 //=============================================================================
1800 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1801
1802 int ConstantTable::calculate_table_base_offset() const {
1803 return 0; // absolute addressing, no offset
1804 }
1805
1806 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1807 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1808 ShouldNotReachHere();
1809 }
1810
1811 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1812 // Empty encoding
1813 }
1814
1815 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1816 return 0;
1817 }
1818
1819 #ifndef PRODUCT
1820 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1821 st->print("# MachConstantBaseNode (empty encoding)");
1822 }
1823 #endif
1824
1825
1826 //=============================================================================
1827 #ifndef PRODUCT
1828 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1829 Compile* C = ra_->C;
1830
1831 int framesize = C->output()->frame_size_in_bytes();
1832 int bangsize = C->output()->bang_size_in_bytes();
1833 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1834 // Remove wordSize for return addr which is already pushed.
1835 framesize -= wordSize;
1836
1837 if (C->output()->need_stack_bang(bangsize)) {
1838 framesize -= wordSize;
1839 st->print("# stack bang (%d bytes)", bangsize);
1840 st->print("\n\t");
1841 st->print("pushq rbp\t# Save rbp");
1842 if (PreserveFramePointer) {
1843 st->print("\n\t");
1844 st->print("movq rbp, rsp\t# Save the caller's SP into rbp");
1845 }
1846 if (framesize) {
1847 st->print("\n\t");
1848 st->print("subq rsp, #%d\t# Create frame",framesize);
1849 }
1850 } else {
1851 st->print("subq rsp, #%d\t# Create frame",framesize);
1852 st->print("\n\t");
1853 framesize -= wordSize;
1854 st->print("movq [rsp + #%d], rbp\t# Save rbp",framesize);
1855 if (PreserveFramePointer) {
1856 st->print("\n\t");
1857 st->print("movq rbp, rsp\t# Save the caller's SP into rbp");
1858 if (framesize > 0) {
1859 st->print("\n\t");
1860 st->print("addq rbp, #%d", framesize);
1861 }
1862 }
1863 }
1864
1865 if (VerifyStackAtCalls) {
1866 st->print("\n\t");
1867 framesize -= wordSize;
1868 st->print("movq [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
1869 #ifdef ASSERT
1870 st->print("\n\t");
1871 st->print("# stack alignment check");
1872 #endif
1873 }
1874 if (C->stub_function() != nullptr) {
1875 st->print("\n\t");
1876 st->print("cmpl [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
1877 st->print("\n\t");
1878 st->print("je fast_entry\t");
1879 st->print("\n\t");
1880 st->print("call #nmethod_entry_barrier_stub\t");
1881 st->print("\n\tfast_entry:");
1882 }
1883 st->cr();
1884 }
1885 #endif
1886
1887 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1888 Compile* C = ra_->C;
1889
1890 int framesize = C->output()->frame_size_in_bytes();
1891 int bangsize = C->output()->bang_size_in_bytes();
1892
1893 if (C->clinit_barrier_on_entry()) {
1894 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
1895 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1896
1897 Label L_skip_barrier;
1898 Register klass = rscratch1;
1899
1900 __ mov_metadata(klass, C->method()->holder()->constant_encoding());
1901 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
1902
1903 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
1904
1905 __ bind(L_skip_barrier);
1906 }
1907
1908 __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
1909
1910 C->output()->set_frame_complete(__ offset());
1911
1912 if (C->has_mach_constant_base_node()) {
1913 // NOTE: We set the table base offset here because users might be
1914 // emitted before MachConstantBaseNode.
1915 ConstantTable& constant_table = C->output()->constant_table();
1916 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1917 }
1918 }
1919
1920 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1921 {
1922 return MachNode::size(ra_); // too many variables; just compute it
1923 // the hard way
1924 }
1925
1926 int MachPrologNode::reloc() const
1927 {
1928 return 0; // a large enough number
1929 }
1930
1931 //=============================================================================
1932 #ifndef PRODUCT
1933 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1934 {
1935 Compile* C = ra_->C;
1936 if (generate_vzeroupper(C)) {
1937 st->print("vzeroupper");
1938 st->cr(); st->print("\t");
1939 }
1940
1941 int framesize = C->output()->frame_size_in_bytes();
1942 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1943 // Remove word for return adr already pushed
1944 // and RBP
1945 framesize -= 2*wordSize;
1946
1947 if (framesize) {
1948 st->print_cr("addq rsp, %d\t# Destroy frame", framesize);
1949 st->print("\t");
1950 }
1951
1952 st->print_cr("popq rbp");
1953 if (do_polling() && C->is_method_compilation()) {
1954 st->print("\t");
1955 st->print_cr("cmpq rsp, poll_offset[r15_thread] \n\t"
1956 "ja #safepoint_stub\t"
1957 "# Safepoint: poll for GC");
1958 }
1959 }
1960 #endif
1961
1962 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
1963 {
1964 Compile* C = ra_->C;
1965
1966 if (generate_vzeroupper(C)) {
1967 // Clear upper bits of YMM registers when current compiled code uses
1968 // wide vectors to avoid AVX <-> SSE transition penalty during call.
1969 __ vzeroupper();
1970 }
1971
1972 int framesize = C->output()->frame_size_in_bytes();
1973 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1974 // Remove word for return adr already pushed
1975 // and RBP
1976 framesize -= 2*wordSize;
1977
1978 // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
1979
1980 if (framesize) {
1981 __ addq(rsp, framesize);
1982 }
1983
1984 __ popq(rbp);
1985
1986 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1987 __ reserved_stack_check();
1988 }
1989
1990 if (do_polling() && C->is_method_compilation()) {
1991 Label dummy_label;
1992 Label* code_stub = &dummy_label;
1993 if (!C->output()->in_scratch_emit_size()) {
1994 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1995 C->output()->add_stub(stub);
1996 code_stub = &stub->entry();
1997 }
1998 __ relocate(relocInfo::poll_return_type);
1999 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
2000 }
2001 }
2002
2003 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
2004 {
2005 return MachNode::size(ra_); // too many variables; just compute it
2006 // the hard way
2007 }
2008
2009 int MachEpilogNode::reloc() const
2010 {
2011 return 2; // a large enough number
2012 }
2013
2014 const Pipeline* MachEpilogNode::pipeline() const
2015 {
2016 return MachNode::pipeline_class();
2017 }
2018
2019 //=============================================================================
2020
2021 enum RC {
2022 rc_bad,
2023 rc_int,
2024 rc_kreg,
2025 rc_float,
2026 rc_stack
2027 };
2028
2029 static enum RC rc_class(OptoReg::Name reg)
2030 {
2031 if( !OptoReg::is_valid(reg) ) return rc_bad;
2032
2033 if (OptoReg::is_stack(reg)) return rc_stack;
2034
2035 VMReg r = OptoReg::as_VMReg(reg);
2036
2037 if (r->is_Register()) return rc_int;
2038
2039 if (r->is_KRegister()) return rc_kreg;
2040
2041 assert(r->is_XMMRegister(), "must be");
2042 return rc_float;
2043 }
2044
2045 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
2046 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
2047 int src_hi, int dst_hi, uint ireg, outputStream* st);
2048
2049 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
2050 int stack_offset, int reg, uint ireg, outputStream* st);
2051
2052 static void vec_stack_to_stack_helper(C2_MacroAssembler *masm, int src_offset,
2053 int dst_offset, uint ireg, outputStream* st) {
2054 if (masm) {
2055 switch (ireg) {
2056 case Op_VecS:
2057 __ movq(Address(rsp, -8), rax);
2058 __ movl(rax, Address(rsp, src_offset));
2059 __ movl(Address(rsp, dst_offset), rax);
2060 __ movq(rax, Address(rsp, -8));
2061 break;
2062 case Op_VecD:
2063 __ pushq(Address(rsp, src_offset));
2064 __ popq (Address(rsp, dst_offset));
2065 break;
2066 case Op_VecX:
2067 __ pushq(Address(rsp, src_offset));
2068 __ popq (Address(rsp, dst_offset));
2069 __ pushq(Address(rsp, src_offset+8));
2070 __ popq (Address(rsp, dst_offset+8));
2071 break;
2072 case Op_VecY:
2073 __ vmovdqu(Address(rsp, -32), xmm0);
2074 __ vmovdqu(xmm0, Address(rsp, src_offset));
2075 __ vmovdqu(Address(rsp, dst_offset), xmm0);
2076 __ vmovdqu(xmm0, Address(rsp, -32));
2077 break;
2078 case Op_VecZ:
2079 __ evmovdquq(Address(rsp, -64), xmm0, 2);
2080 __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
2081 __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
2082 __ evmovdquq(xmm0, Address(rsp, -64), 2);
2083 break;
2084 default:
2085 ShouldNotReachHere();
2086 }
2087 #ifndef PRODUCT
2088 } else {
2089 switch (ireg) {
2090 case Op_VecS:
2091 st->print("movq [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
2092 "movl rax, [rsp + #%d]\n\t"
2093 "movl [rsp + #%d], rax\n\t"
2094 "movq rax, [rsp - #8]",
2095 src_offset, dst_offset);
2096 break;
2097 case Op_VecD:
2098 st->print("pushq [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
2099 "popq [rsp + #%d]",
2100 src_offset, dst_offset);
2101 break;
2102 case Op_VecX:
2103 st->print("pushq [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
2104 "popq [rsp + #%d]\n\t"
2105 "pushq [rsp + #%d]\n\t"
2106 "popq [rsp + #%d]",
2107 src_offset, dst_offset, src_offset+8, dst_offset+8);
2108 break;
2109 case Op_VecY:
2110 st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
2111 "vmovdqu xmm0, [rsp + #%d]\n\t"
2112 "vmovdqu [rsp + #%d], xmm0\n\t"
2113 "vmovdqu xmm0, [rsp - #32]",
2114 src_offset, dst_offset);
2115 break;
2116 case Op_VecZ:
2117 st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
2118 "vmovdqu xmm0, [rsp + #%d]\n\t"
2119 "vmovdqu [rsp + #%d], xmm0\n\t"
2120 "vmovdqu xmm0, [rsp - #64]",
2121 src_offset, dst_offset);
2122 break;
2123 default:
2124 ShouldNotReachHere();
2125 }
2126 #endif
2127 }
2128 }
2129
2130 uint MachSpillCopyNode::implementation(C2_MacroAssembler* masm,
2131 PhaseRegAlloc* ra_,
2132 bool do_size,
2133 outputStream* st) const {
2134 assert(masm != nullptr || st != nullptr, "sanity");
2135 // Get registers to move
2136 OptoReg::Name src_second = ra_->get_reg_second(in(1));
2137 OptoReg::Name src_first = ra_->get_reg_first(in(1));
2138 OptoReg::Name dst_second = ra_->get_reg_second(this);
2139 OptoReg::Name dst_first = ra_->get_reg_first(this);
2140
2141 enum RC src_second_rc = rc_class(src_second);
2142 enum RC src_first_rc = rc_class(src_first);
2143 enum RC dst_second_rc = rc_class(dst_second);
2144 enum RC dst_first_rc = rc_class(dst_first);
2145
2146 assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
2147 "must move at least 1 register" );
2148
2149 if (src_first == dst_first && src_second == dst_second) {
2150 // Self copy, no move
2151 return 0;
2152 }
2153 if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_pvectmask() == nullptr) {
2154 uint ireg = ideal_reg();
2155 assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
2156 assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
2157 if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
2158 // mem -> mem
2159 int src_offset = ra_->reg2offset(src_first);
2160 int dst_offset = ra_->reg2offset(dst_first);
2161 vec_stack_to_stack_helper(masm, src_offset, dst_offset, ireg, st);
2162 } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
2163 vec_mov_helper(masm, src_first, dst_first, src_second, dst_second, ireg, st);
2164 } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
2165 int stack_offset = ra_->reg2offset(dst_first);
2166 vec_spill_helper(masm, false, stack_offset, src_first, ireg, st);
2167 } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
2168 int stack_offset = ra_->reg2offset(src_first);
2169 vec_spill_helper(masm, true, stack_offset, dst_first, ireg, st);
2170 } else {
2171 ShouldNotReachHere();
2172 }
2173 return 0;
2174 }
2175 if (src_first_rc == rc_stack) {
2176 // mem ->
2177 if (dst_first_rc == rc_stack) {
2178 // mem -> mem
2179 assert(src_second != dst_first, "overlap");
2180 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2181 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2182 // 64-bit
2183 int src_offset = ra_->reg2offset(src_first);
2184 int dst_offset = ra_->reg2offset(dst_first);
2185 if (masm) {
2186 __ pushq(Address(rsp, src_offset));
2187 __ popq (Address(rsp, dst_offset));
2188 #ifndef PRODUCT
2189 } else {
2190 st->print("pushq [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
2191 "popq [rsp + #%d]",
2192 src_offset, dst_offset);
2193 #endif
2194 }
2195 } else {
2196 // 32-bit
2197 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2198 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2199 // No pushl/popl, so:
2200 int src_offset = ra_->reg2offset(src_first);
2201 int dst_offset = ra_->reg2offset(dst_first);
2202 if (masm) {
2203 __ movq(Address(rsp, -8), rax);
2204 __ movl(rax, Address(rsp, src_offset));
2205 __ movl(Address(rsp, dst_offset), rax);
2206 __ movq(rax, Address(rsp, -8));
2207 #ifndef PRODUCT
2208 } else {
2209 st->print("movq [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
2210 "movl rax, [rsp + #%d]\n\t"
2211 "movl [rsp + #%d], rax\n\t"
2212 "movq rax, [rsp - #8]",
2213 src_offset, dst_offset);
2214 #endif
2215 }
2216 }
2217 return 0;
2218 } else if (dst_first_rc == rc_int) {
2219 // mem -> gpr
2220 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2221 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2222 // 64-bit
2223 int offset = ra_->reg2offset(src_first);
2224 if (masm) {
2225 __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2226 #ifndef PRODUCT
2227 } else {
2228 st->print("movq %s, [rsp + #%d]\t# spill",
2229 Matcher::regName[dst_first],
2230 offset);
2231 #endif
2232 }
2233 } else {
2234 // 32-bit
2235 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2236 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2237 int offset = ra_->reg2offset(src_first);
2238 if (masm) {
2239 __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2240 #ifndef PRODUCT
2241 } else {
2242 st->print("movl %s, [rsp + #%d]\t# spill",
2243 Matcher::regName[dst_first],
2244 offset);
2245 #endif
2246 }
2247 }
2248 return 0;
2249 } else if (dst_first_rc == rc_float) {
2250 // mem-> xmm
2251 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2252 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2253 // 64-bit
2254 int offset = ra_->reg2offset(src_first);
2255 if (masm) {
2256 __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2257 #ifndef PRODUCT
2258 } else {
2259 st->print("%s %s, [rsp + #%d]\t# spill",
2260 UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
2261 Matcher::regName[dst_first],
2262 offset);
2263 #endif
2264 }
2265 } else {
2266 // 32-bit
2267 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2268 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2269 int offset = ra_->reg2offset(src_first);
2270 if (masm) {
2271 __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2272 #ifndef PRODUCT
2273 } else {
2274 st->print("movss %s, [rsp + #%d]\t# spill",
2275 Matcher::regName[dst_first],
2276 offset);
2277 #endif
2278 }
2279 }
2280 return 0;
2281 } else if (dst_first_rc == rc_kreg) {
2282 // mem -> kreg
2283 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2284 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2285 // 64-bit
2286 int offset = ra_->reg2offset(src_first);
2287 if (masm) {
2288 __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2289 #ifndef PRODUCT
2290 } else {
2291 st->print("kmovq %s, [rsp + #%d]\t# spill",
2292 Matcher::regName[dst_first],
2293 offset);
2294 #endif
2295 }
2296 }
2297 return 0;
2298 }
2299 } else if (src_first_rc == rc_int) {
2300 // gpr ->
2301 if (dst_first_rc == rc_stack) {
2302 // gpr -> mem
2303 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2304 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2305 // 64-bit
2306 int offset = ra_->reg2offset(dst_first);
2307 if (masm) {
2308 __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
2309 #ifndef PRODUCT
2310 } else {
2311 st->print("movq [rsp + #%d], %s\t# spill",
2312 offset,
2313 Matcher::regName[src_first]);
2314 #endif
2315 }
2316 } else {
2317 // 32-bit
2318 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2319 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2320 int offset = ra_->reg2offset(dst_first);
2321 if (masm) {
2322 __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
2323 #ifndef PRODUCT
2324 } else {
2325 st->print("movl [rsp + #%d], %s\t# spill",
2326 offset,
2327 Matcher::regName[src_first]);
2328 #endif
2329 }
2330 }
2331 return 0;
2332 } else if (dst_first_rc == rc_int) {
2333 // gpr -> gpr
2334 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2335 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2336 // 64-bit
2337 if (masm) {
2338 __ movq(as_Register(Matcher::_regEncode[dst_first]),
2339 as_Register(Matcher::_regEncode[src_first]));
2340 #ifndef PRODUCT
2341 } else {
2342 st->print("movq %s, %s\t# spill",
2343 Matcher::regName[dst_first],
2344 Matcher::regName[src_first]);
2345 #endif
2346 }
2347 return 0;
2348 } else {
2349 // 32-bit
2350 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2351 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2352 if (masm) {
2353 __ movl(as_Register(Matcher::_regEncode[dst_first]),
2354 as_Register(Matcher::_regEncode[src_first]));
2355 #ifndef PRODUCT
2356 } else {
2357 st->print("movl %s, %s\t# spill",
2358 Matcher::regName[dst_first],
2359 Matcher::regName[src_first]);
2360 #endif
2361 }
2362 return 0;
2363 }
2364 } else if (dst_first_rc == rc_float) {
2365 // gpr -> xmm
2366 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2367 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2368 // 64-bit
2369 if (masm) {
2370 __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
2371 #ifndef PRODUCT
2372 } else {
2373 st->print("movdq %s, %s\t# spill",
2374 Matcher::regName[dst_first],
2375 Matcher::regName[src_first]);
2376 #endif
2377 }
2378 } else {
2379 // 32-bit
2380 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2381 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2382 if (masm) {
2383 __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
2384 #ifndef PRODUCT
2385 } else {
2386 st->print("movdl %s, %s\t# spill",
2387 Matcher::regName[dst_first],
2388 Matcher::regName[src_first]);
2389 #endif
2390 }
2391 }
2392 return 0;
2393 } else if (dst_first_rc == rc_kreg) {
2394 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2395 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2396 // 64-bit
2397 if (masm) {
2398 __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
2399 #ifndef PRODUCT
2400 } else {
2401 st->print("kmovq %s, %s\t# spill",
2402 Matcher::regName[dst_first],
2403 Matcher::regName[src_first]);
2404 #endif
2405 }
2406 }
2407 Unimplemented();
2408 return 0;
2409 }
2410 } else if (src_first_rc == rc_float) {
2411 // xmm ->
2412 if (dst_first_rc == rc_stack) {
2413 // xmm -> mem
2414 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2415 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2416 // 64-bit
2417 int offset = ra_->reg2offset(dst_first);
2418 if (masm) {
2419 __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
2420 #ifndef PRODUCT
2421 } else {
2422 st->print("movsd [rsp + #%d], %s\t# spill",
2423 offset,
2424 Matcher::regName[src_first]);
2425 #endif
2426 }
2427 } else {
2428 // 32-bit
2429 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2430 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2431 int offset = ra_->reg2offset(dst_first);
2432 if (masm) {
2433 __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
2434 #ifndef PRODUCT
2435 } else {
2436 st->print("movss [rsp + #%d], %s\t# spill",
2437 offset,
2438 Matcher::regName[src_first]);
2439 #endif
2440 }
2441 }
2442 return 0;
2443 } else if (dst_first_rc == rc_int) {
2444 // xmm -> gpr
2445 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2446 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2447 // 64-bit
2448 if (masm) {
2449 __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2450 #ifndef PRODUCT
2451 } else {
2452 st->print("movdq %s, %s\t# spill",
2453 Matcher::regName[dst_first],
2454 Matcher::regName[src_first]);
2455 #endif
2456 }
2457 } else {
2458 // 32-bit
2459 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2460 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2461 if (masm) {
2462 __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2463 #ifndef PRODUCT
2464 } else {
2465 st->print("movdl %s, %s\t# spill",
2466 Matcher::regName[dst_first],
2467 Matcher::regName[src_first]);
2468 #endif
2469 }
2470 }
2471 return 0;
2472 } else if (dst_first_rc == rc_float) {
2473 // xmm -> xmm
2474 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2475 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2476 // 64-bit
2477 if (masm) {
2478 __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2479 #ifndef PRODUCT
2480 } else {
2481 st->print("%s %s, %s\t# spill",
2482 UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
2483 Matcher::regName[dst_first],
2484 Matcher::regName[src_first]);
2485 #endif
2486 }
2487 } else {
2488 // 32-bit
2489 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2490 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2491 if (masm) {
2492 __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2493 #ifndef PRODUCT
2494 } else {
2495 st->print("%s %s, %s\t# spill",
2496 UseXmmRegToRegMoveAll ? "movaps" : "movss ",
2497 Matcher::regName[dst_first],
2498 Matcher::regName[src_first]);
2499 #endif
2500 }
2501 }
2502 return 0;
2503 } else if (dst_first_rc == rc_kreg) {
2504 assert(false, "Illegal spilling");
2505 return 0;
2506 }
2507 } else if (src_first_rc == rc_kreg) {
2508 if (dst_first_rc == rc_stack) {
2509 // mem -> kreg
2510 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2511 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2512 // 64-bit
2513 int offset = ra_->reg2offset(dst_first);
2514 if (masm) {
2515 __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
2516 #ifndef PRODUCT
2517 } else {
2518 st->print("kmovq [rsp + #%d] , %s\t# spill",
2519 offset,
2520 Matcher::regName[src_first]);
2521 #endif
2522 }
2523 }
2524 return 0;
2525 } else if (dst_first_rc == rc_int) {
2526 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2527 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2528 // 64-bit
2529 if (masm) {
2530 __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
2531 #ifndef PRODUCT
2532 } else {
2533 st->print("kmovq %s, %s\t# spill",
2534 Matcher::regName[dst_first],
2535 Matcher::regName[src_first]);
2536 #endif
2537 }
2538 }
2539 Unimplemented();
2540 return 0;
2541 } else if (dst_first_rc == rc_kreg) {
2542 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2543 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2544 // 64-bit
2545 if (masm) {
2546 __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
2547 #ifndef PRODUCT
2548 } else {
2549 st->print("kmovq %s, %s\t# spill",
2550 Matcher::regName[dst_first],
2551 Matcher::regName[src_first]);
2552 #endif
2553 }
2554 }
2555 return 0;
2556 } else if (dst_first_rc == rc_float) {
2557 assert(false, "Illegal spill");
2558 return 0;
2559 }
2560 }
2561
2562 assert(0," foo ");
2563 Unimplemented();
2564 return 0;
2565 }
2566
2567 #ifndef PRODUCT
2568 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
2569 implementation(nullptr, ra_, false, st);
2570 }
2571 #endif
2572
2573 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2574 implementation(masm, ra_, false, nullptr);
2575 }
2576
2577 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2578 return MachNode::size(ra_);
2579 }
2580
2581 //=============================================================================
2582 #ifndef PRODUCT
2583 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2584 {
2585 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2586 int reg = ra_->get_reg_first(this);
2587 st->print("leaq %s, [rsp + #%d]\t# box lock",
2588 Matcher::regName[reg], offset);
2589 }
2590 #endif
2591
2592 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2593 {
2594 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2595 int reg = ra_->get_encode(this);
2596
2597 __ lea(as_Register(reg), Address(rsp, offset));
2598 }
2599
2600 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
2601 {
2602 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2603 if (ra_->get_encode(this) > 15) {
2604 return (offset < 0x80) ? 6 : 9; // REX2
2605 } else {
2606 return (offset < 0x80) ? 5 : 8; // REX
2607 }
2608 }
2609
2610 //=============================================================================
2611 #ifndef PRODUCT
2612 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2613 {
2614 st->print_cr("movl rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2615 st->print_cr("\tcmpl rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
2616 st->print_cr("\tjne SharedRuntime::_ic_miss_stub");
2617 }
2618 #endif
2619
2620 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2621 {
2622 __ ic_check(InteriorEntryAlignment);
2623 }
2624
2625 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2626 {
2627 return MachNode::size(ra_); // too many variables; just compute it
2628 // the hard way
2629 }
2630
2631
2632 //=============================================================================
2633
2634 bool Matcher::supports_vector_calling_convention(void) {
2635 return EnableVectorSupport;
2636 }
2637
2638 static bool is_ndd_demotable_opr1(const MachNode* mdef) {
2639 return ((mdef->flags() & Node::PD::Flag_ndd_demotable_opr1) != 0);
2640 }
2641
2642 static bool is_ndd_demotable_opr2(const MachNode* mdef) {
2643 return ((mdef->flags() & Node::PD::Flag_ndd_demotable_opr2) != 0);
2644 }
2645
2646 #ifdef ASSERT
2647 static bool is_ndd_demotable(const MachNode* mdef) {
2648 return (is_ndd_demotable_opr1(mdef) || is_ndd_demotable_opr2(mdef));
2649 }
2650 #endif
2651
2652 bool Matcher::is_register_biasing_candidate(const MachNode* mdef,
2653 int oper_index) {
2654 if (mdef == nullptr) {
2655 return false;
2656 }
2657
2658 if (mdef->num_opnds() <= oper_index || mdef->operand_index(oper_index) < 0 ||
2659 mdef->in(mdef->operand_index(oper_index)) == nullptr) {
2660 assert(oper_index != 1 || !is_ndd_demotable_opr1(mdef), "%s", mdef->Name());
2661 assert(oper_index != 2 || !is_ndd_demotable_opr2(mdef), "%s", mdef->Name());
2662 return false;
2663 }
2664
2665 // Complex memory operand covers multiple incoming edges needed for
2666 // address computation. Biasing def towards any address component will not
2667 // result in NDD demotion by assembler.
2668 if (mdef->operand_num_edges(oper_index) != 1) {
2669 return false;
2670 }
2671
2672 // Demotion candidate must be register mask compatible with definition.
2673 const RegMask& oper_mask = mdef->in_RegMask(mdef->operand_index(oper_index));
2674 if (!oper_mask.overlap(mdef->out_RegMask())) {
2675 assert(!is_ndd_demotable(mdef), "%s", mdef->Name());
2676 return false;
2677 }
2678
2679 switch (oper_index) {
2680 // First operand of MachNode corresponding to Intel APX NDD selection
2681 // pattern can share its assigned register with definition operand if
2682 // their live ranges do not overlap. In such a scenario we can demote
2683 // it to legacy map0/map1 instruction by replacing its 4-byte extended
2684 // EVEX prefix with shorter REX/REX2 encoding. Demotion candidates
2685 // are decorated with a special flag by instruction selector.
2686 case 1:
2687 return is_ndd_demotable_opr1(mdef);
2688
2689 // Definition operand of commutative operation can be biased towards second
2690 // operand.
2691 case 2:
2692 return is_ndd_demotable_opr2(mdef);
2693
2694 // Current scheme only selects up to two biasing candidates
2695 default:
2696 assert(false, "unhandled operand index: %s", mdef->Name());
2697 break;
2698 }
2699
2700 return false;
2701 }
2702
2703 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2704 assert(EnableVectorSupport, "sanity");
2705 int lo = XMM0_num;
2706 int hi = XMM0b_num;
2707 if (ideal_reg == Op_VecX) hi = XMM0d_num;
2708 else if (ideal_reg == Op_VecY) hi = XMM0h_num;
2709 else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
2710 return OptoRegPair(hi, lo);
2711 }
2712
2713 // Is this branch offset short enough that a short branch can be used?
2714 //
2715 // NOTE: If the platform does not provide any short branch variants, then
2716 // this method should return false for offset 0.
2717 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2718 // The passed offset is relative to address of the branch.
2719 // On 86 a branch displacement is calculated relative to address
2720 // of a next instruction.
2721 offset -= br_size;
2722
2723 // the short version of jmpConUCF2 contains multiple branches,
2724 // making the reach slightly less
2725 if (rule == jmpConUCF2_rule)
2726 return (-126 <= offset && offset <= 125);
2727 return (-128 <= offset && offset <= 127);
2728 }
2729
2730 #ifdef ASSERT
2731 // Return whether or not this register is ever used as an argument.
2732 bool Matcher::can_be_java_arg(int reg)
2733 {
2734 return
2735 reg == RDI_num || reg == RDI_H_num ||
2736 reg == RSI_num || reg == RSI_H_num ||
2737 reg == RDX_num || reg == RDX_H_num ||
2738 reg == RCX_num || reg == RCX_H_num ||
2739 reg == R8_num || reg == R8_H_num ||
2740 reg == R9_num || reg == R9_H_num ||
2741 reg == R12_num || reg == R12_H_num ||
2742 reg == XMM0_num || reg == XMM0b_num ||
2743 reg == XMM1_num || reg == XMM1b_num ||
2744 reg == XMM2_num || reg == XMM2b_num ||
2745 reg == XMM3_num || reg == XMM3b_num ||
2746 reg == XMM4_num || reg == XMM4b_num ||
2747 reg == XMM5_num || reg == XMM5b_num ||
2748 reg == XMM6_num || reg == XMM6b_num ||
2749 reg == XMM7_num || reg == XMM7b_num;
2750 }
2751 #endif
2752
2753 uint Matcher::int_pressure_limit()
2754 {
2755 return (INTPRESSURE == -1) ? _INT_REG_mask.size() : INTPRESSURE;
2756 }
2757
2758 uint Matcher::float_pressure_limit()
2759 {
2760 // After experiment around with different values, the following default threshold
2761 // works best for LCM's register pressure scheduling on x64.
2762 uint dec_count = VM_Version::supports_evex() ? 4 : 2;
2763 uint default_float_pressure_threshold = _FLOAT_REG_mask.size() - dec_count;
2764 return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
2765 }
2766
2767 // Register for the first projection of an int pair
2768 const RegMask& Matcher::firstI_proj_mask() {
2769 return INT_RAX_REG_mask();
2770 }
2771
2772 // Register for the second projection of an int pair
2773 const RegMask& Matcher::secondI_proj_mask() {
2774 return INT_RDX_REG_mask();
2775 }
2776
2777 // Register for the first projection of a long pair
2778 const RegMask& Matcher::firstL_proj_mask() {
2779 return LONG_RAX_REG_mask();
2780 }
2781
2782 // Register for the second projection of a long pair
2783 const RegMask& Matcher::secondL_proj_mask() {
2784 return LONG_RDX_REG_mask();
2785 }
2786
2787 %}
2788
2789 source_hpp %{
2790 // Header information of the source block.
2791 // Method declarations/definitions which are used outside
2792 // the ad-scope can conveniently be defined here.
2793 //
2794 // To keep related declarations/definitions/uses close together,
2795 // we switch between source %{ }% and source_hpp %{ }% freely as needed.
2796
2797 #include "runtime/vm_version.hpp"
2798
2799 class NativeJump;
2800
2801 class CallStubImpl {
2802
2803 //--------------------------------------------------------------
2804 //---< Used for optimization in Compile::shorten_branches >---
2805 //--------------------------------------------------------------
2806
2807 public:
2808 // Size of call trampoline stub.
2809 static uint size_call_trampoline() {
2810 return 0; // no call trampolines on this platform
2811 }
2812
2813 // number of relocations needed by a call trampoline stub
2814 static uint reloc_call_trampoline() {
2815 return 0; // no call trampolines on this platform
2816 }
2817 };
2818
2819 class HandlerImpl {
2820
2821 public:
2822
2823 static int emit_deopt_handler(C2_MacroAssembler* masm);
2824
2825 static uint size_deopt_handler() {
2826 // one call and one jmp.
2827 return 7;
2828 }
2829 };
2830
2831 inline Assembler::AvxVectorLen vector_length_encoding(int bytes) {
2832 switch(bytes) {
2833 case 4: // fall-through
2834 case 8: // fall-through
2835 case 16: return Assembler::AVX_128bit;
2836 case 32: return Assembler::AVX_256bit;
2837 case 64: return Assembler::AVX_512bit;
2838
2839 default: {
2840 ShouldNotReachHere();
2841 return Assembler::AVX_NoVec;
2842 }
2843 }
2844 }
2845
2846 static inline Assembler::AvxVectorLen vector_length_encoding(const Node* n) {
2847 return vector_length_encoding(Matcher::vector_length_in_bytes(n));
2848 }
2849
2850 static inline Assembler::AvxVectorLen vector_length_encoding(const MachNode* use, MachOper* opnd) {
2851 uint def_idx = use->operand_index(opnd);
2852 Node* def = use->in(def_idx);
2853 return vector_length_encoding(def);
2854 }
2855
2856 static inline bool is_vector_popcount_predicate(BasicType bt) {
2857 return (is_subword_type(bt) && VM_Version::supports_avx512_bitalg()) ||
2858 (is_non_subword_integral_type(bt) && VM_Version::supports_avx512_vpopcntdq());
2859 }
2860
2861 static inline bool is_clz_non_subword_predicate_evex(BasicType bt, int vlen_bytes) {
2862 return is_non_subword_integral_type(bt) && VM_Version::supports_avx512cd() &&
2863 (VM_Version::supports_avx512vl() || vlen_bytes == 64);
2864 }
2865
2866 class Node::PD {
2867 public:
2868 enum NodeFlags : uint64_t {
2869 Flag_intel_jcc_erratum = Node::_last_flag << 1,
2870 Flag_sets_carry_flag = Node::_last_flag << 2,
2871 Flag_sets_parity_flag = Node::_last_flag << 3,
2872 Flag_sets_zero_flag = Node::_last_flag << 4,
2873 Flag_sets_overflow_flag = Node::_last_flag << 5,
2874 Flag_sets_sign_flag = Node::_last_flag << 6,
2875 Flag_clears_carry_flag = Node::_last_flag << 7,
2876 Flag_clears_parity_flag = Node::_last_flag << 8,
2877 Flag_clears_zero_flag = Node::_last_flag << 9,
2878 Flag_clears_overflow_flag = Node::_last_flag << 10,
2879 Flag_clears_sign_flag = Node::_last_flag << 11,
2880 Flag_ndd_demotable_opr1 = Node::_last_flag << 12,
2881 Flag_ndd_demotable_opr2 = Node::_last_flag << 13,
2882 _last_flag = Flag_ndd_demotable_opr2
2883 };
2884 };
2885
2886 %} // end source_hpp
2887
2888 source %{
2889
2890 #include "opto/addnode.hpp"
2891 #include "c2_intelJccErratum_x86.hpp"
2892
2893 void PhaseOutput::pd_perform_mach_node_analysis() {
2894 if (VM_Version::has_intel_jcc_erratum()) {
2895 int extra_padding = IntelJccErratum::tag_affected_machnodes(C, C->cfg(), C->regalloc());
2896 _buf_sizes._code += extra_padding;
2897 }
2898 }
2899
2900 int MachNode::pd_alignment_required() const {
2901 if (VM_Version::has_intel_jcc_erratum() && IntelJccErratum::is_jcc_erratum_branch(this)) {
2902 // Conservatively add worst case padding. We assume that relocInfo::addr_unit() is 1 on x86.
2903 return IntelJccErratum::largest_jcc_size() + 1;
2904 } else {
2905 return 1;
2906 }
2907 }
2908
2909 int MachNode::compute_padding(int current_offset) const {
2910 if (flags() & Node::PD::Flag_intel_jcc_erratum) {
2911 Compile* C = Compile::current();
2912 PhaseOutput* output = C->output();
2913 Block* block = output->block();
2914 int index = output->index();
2915 return IntelJccErratum::compute_padding(current_offset, this, block, index, C->regalloc());
2916 } else {
2917 return 0;
2918 }
2919 }
2920
2921 // Emit deopt handler code.
2922 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) {
2923
2924 // Note that the code buffer's insts_mark is always relative to insts.
2925 // That's why we must use the macroassembler to generate a handler.
2926 address base = __ start_a_stub(size_deopt_handler());
2927 if (base == nullptr) {
2928 ciEnv::current()->record_failure("CodeCache is full");
2929 return 0; // CodeBuffer::expand failed
2930 }
2931 int offset = __ offset();
2932
2933 Label start;
2934 __ bind(start);
2935
2936 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2937
2938 int entry_offset = __ offset();
2939
2940 __ jmp(start);
2941
2942 assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow %d", (__ offset() - offset));
2943 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2944 "out of bounds read in post-call NOP check");
2945 __ end_a_stub();
2946 return entry_offset;
2947 }
2948
2949 static Assembler::Width widthForType(BasicType bt) {
2950 if (bt == T_BYTE) {
2951 return Assembler::B;
2952 } else if (bt == T_SHORT) {
2953 return Assembler::W;
2954 } else if (bt == T_INT) {
2955 return Assembler::D;
2956 } else {
2957 assert(bt == T_LONG, "not a long: %s", type2name(bt));
2958 return Assembler::Q;
2959 }
2960 }
2961
2962 //=============================================================================
2963
2964 // Float masks come from different places depending on platform.
2965 static address float_signmask() { return StubRoutines::x86::float_sign_mask(); }
2966 static address float_signflip() { return StubRoutines::x86::float_sign_flip(); }
2967 static address double_signmask() { return StubRoutines::x86::double_sign_mask(); }
2968 static address double_signflip() { return StubRoutines::x86::double_sign_flip(); }
2969 static address vector_short_to_byte_mask() { return StubRoutines::x86::vector_short_to_byte_mask(); }
2970 static address vector_int_to_byte_mask() { return StubRoutines::x86::vector_int_to_byte_mask(); }
2971 static address vector_byte_perm_mask() { return StubRoutines::x86::vector_byte_perm_mask(); }
2972 static address vector_long_sign_mask() { return StubRoutines::x86::vector_long_sign_mask(); }
2973 static address vector_all_bits_set() { return StubRoutines::x86::vector_all_bits_set(); }
2974 static address vector_int_mask_cmp_bits() { return StubRoutines::x86::vector_int_mask_cmp_bits(); }
2975 static address vector_int_to_short_mask() { return StubRoutines::x86::vector_int_to_short_mask(); }
2976 static address vector_byte_shufflemask() { return StubRoutines::x86::vector_byte_shuffle_mask(); }
2977 static address vector_short_shufflemask() { return StubRoutines::x86::vector_short_shuffle_mask(); }
2978 static address vector_int_shufflemask() { return StubRoutines::x86::vector_int_shuffle_mask(); }
2979 static address vector_long_shufflemask() { return StubRoutines::x86::vector_long_shuffle_mask(); }
2980 static address vector_32_bit_mask() { return StubRoutines::x86::vector_32_bit_mask(); }
2981 static address vector_64_bit_mask() { return StubRoutines::x86::vector_64_bit_mask(); }
2982 static address vector_float_signflip() { return StubRoutines::x86::vector_float_sign_flip();}
2983 static address vector_double_signflip() { return StubRoutines::x86::vector_double_sign_flip();}
2984
2985 //=============================================================================
2986 bool Matcher::match_rule_supported(int opcode) {
2987 if (!has_match_rule(opcode)) {
2988 return false; // no match rule present
2989 }
2990 switch (opcode) {
2991 case Op_AbsVL:
2992 case Op_StoreVectorScatter:
2993 if (UseAVX < 3) {
2994 return false;
2995 }
2996 break;
2997 case Op_PopCountI:
2998 case Op_PopCountL:
2999 if (!UsePopCountInstruction) {
3000 return false;
3001 }
3002 break;
3003 case Op_PopCountVI:
3004 if (UseAVX < 2) {
3005 return false;
3006 }
3007 break;
3008 case Op_CompressV:
3009 case Op_ExpandV:
3010 case Op_PopCountVL:
3011 if (UseAVX < 2) {
3012 return false;
3013 }
3014 break;
3015 case Op_MulVI:
3016 if ((UseSSE < 4) && (UseAVX < 1)) { // only with SSE4_1 or AVX
3017 return false;
3018 }
3019 break;
3020 case Op_MulVL:
3021 if (UseSSE < 4) { // only with SSE4_1 or AVX
3022 return false;
3023 }
3024 break;
3025 case Op_MulReductionVL:
3026 if (VM_Version::supports_avx512dq() == false) {
3027 return false;
3028 }
3029 break;
3030 case Op_AbsVB:
3031 case Op_AbsVS:
3032 case Op_AbsVI:
3033 case Op_AddReductionVI:
3034 case Op_AndReductionV:
3035 case Op_OrReductionV:
3036 case Op_XorReductionV:
3037 if (UseSSE < 3) { // requires at least SSSE3
3038 return false;
3039 }
3040 break;
3041 case Op_MaxHF:
3042 case Op_MinHF:
3043 if (!VM_Version::supports_avx512vlbw()) {
3044 return false;
3045 } // fallthrough
3046 case Op_AddHF:
3047 case Op_DivHF:
3048 case Op_FmaHF:
3049 case Op_MulHF:
3050 case Op_ReinterpretS2HF:
3051 case Op_ReinterpretHF2S:
3052 case Op_SubHF:
3053 case Op_SqrtHF:
3054 if (!VM_Version::supports_avx512_fp16()) {
3055 return false;
3056 }
3057 break;
3058 case Op_VectorLoadShuffle:
3059 case Op_VectorRearrange:
3060 case Op_MulReductionVI:
3061 if (UseSSE < 4) { // requires at least SSE4
3062 return false;
3063 }
3064 break;
3065 case Op_IsInfiniteF:
3066 case Op_IsInfiniteD:
3067 if (!VM_Version::supports_avx512dq()) {
3068 return false;
3069 }
3070 break;
3071 case Op_SqrtVD:
3072 case Op_SqrtVF:
3073 case Op_VectorMaskCmp:
3074 case Op_VectorCastB2X:
3075 case Op_VectorCastS2X:
3076 case Op_VectorCastI2X:
3077 case Op_VectorCastL2X:
3078 case Op_VectorCastF2X:
3079 case Op_VectorCastD2X:
3080 case Op_VectorUCastB2X:
3081 case Op_VectorUCastS2X:
3082 case Op_VectorUCastI2X:
3083 case Op_VectorMaskCast:
3084 if (UseAVX < 1) { // enabled for AVX only
3085 return false;
3086 }
3087 break;
3088 case Op_PopulateIndex:
3089 if (UseAVX < 2) {
3090 return false;
3091 }
3092 break;
3093 case Op_RoundVF:
3094 if (UseAVX < 2) { // enabled for AVX2 only
3095 return false;
3096 }
3097 break;
3098 case Op_RoundVD:
3099 if (UseAVX < 3) {
3100 return false; // enabled for AVX3 only
3101 }
3102 break;
3103 case Op_CompareAndSwapL:
3104 case Op_CompareAndSwapP:
3105 break;
3106 case Op_StrIndexOf:
3107 if (!UseSSE42Intrinsics) {
3108 return false;
3109 }
3110 break;
3111 case Op_StrIndexOfChar:
3112 if (!UseSSE42Intrinsics) {
3113 return false;
3114 }
3115 break;
3116 case Op_OnSpinWait:
3117 if (VM_Version::supports_on_spin_wait() == false) {
3118 return false;
3119 }
3120 break;
3121 case Op_MulVB:
3122 case Op_LShiftVB:
3123 case Op_RShiftVB:
3124 case Op_URShiftVB:
3125 case Op_VectorInsert:
3126 case Op_VectorLoadMask:
3127 case Op_VectorStoreMask:
3128 case Op_VectorBlend:
3129 if (UseSSE < 4) {
3130 return false;
3131 }
3132 break;
3133 case Op_MaxD:
3134 case Op_MaxF:
3135 case Op_MinD:
3136 case Op_MinF:
3137 if (UseAVX < 1) { // enabled for AVX only
3138 return false;
3139 }
3140 break;
3141 case Op_CacheWB:
3142 case Op_CacheWBPreSync:
3143 case Op_CacheWBPostSync:
3144 if (!VM_Version::supports_data_cache_line_flush()) {
3145 return false;
3146 }
3147 break;
3148 case Op_ExtractB:
3149 case Op_ExtractL:
3150 case Op_ExtractI:
3151 case Op_RoundDoubleMode:
3152 if (UseSSE < 4) {
3153 return false;
3154 }
3155 break;
3156 case Op_RoundDoubleModeV:
3157 if (VM_Version::supports_avx() == false) {
3158 return false; // 128bit vroundpd is not available
3159 }
3160 break;
3161 case Op_LoadVectorGather:
3162 case Op_LoadVectorGatherMasked:
3163 if (UseAVX < 2) {
3164 return false;
3165 }
3166 break;
3167 case Op_FmaF:
3168 case Op_FmaD:
3169 case Op_FmaVD:
3170 case Op_FmaVF:
3171 if (!UseFMA) {
3172 return false;
3173 }
3174 break;
3175 case Op_MacroLogicV:
3176 if (UseAVX < 3 || !UseVectorMacroLogic) {
3177 return false;
3178 }
3179 break;
3180
3181 case Op_VectorCmpMasked:
3182 if (!UseCountTrailingZerosInstruction) {
3183 return false;
3184 }
3185 if (UseAVX < 3 || !VM_Version::supports_bmi2()) {
3186 return false;
3187 }
3188 break;
3189 case Op_VectorMaskGen:
3190 if (UseAVX < 3 || !VM_Version::supports_bmi2()) {
3191 return false;
3192 }
3193 break;
3194 case Op_VectorMaskFirstTrue:
3195 case Op_VectorMaskLastTrue:
3196 case Op_VectorMaskTrueCount:
3197 case Op_VectorMaskToLong:
3198 if (UseAVX < 1) {
3199 return false;
3200 }
3201 break;
3202 case Op_RoundF:
3203 case Op_RoundD:
3204 break;
3205 case Op_CopySignD:
3206 case Op_CopySignF:
3207 if (UseAVX < 3) {
3208 return false;
3209 }
3210 if (!VM_Version::supports_avx512vl()) {
3211 return false;
3212 }
3213 break;
3214 case Op_CompressBits:
3215 case Op_ExpandBits:
3216 if (!VM_Version::supports_bmi2()) {
3217 return false;
3218 }
3219 break;
3220 case Op_CompressM:
3221 if (!VM_Version::supports_avx512vl() || !VM_Version::supports_bmi2()) {
3222 return false;
3223 }
3224 break;
3225 case Op_ConvF2HF:
3226 case Op_ConvHF2F:
3227 if (!VM_Version::supports_float16()) {
3228 return false;
3229 }
3230 break;
3231 case Op_VectorCastF2HF:
3232 case Op_VectorCastHF2F:
3233 if (!VM_Version::supports_f16c() && !VM_Version::supports_evex()) {
3234 return false;
3235 }
3236 break;
3237 }
3238 return true; // Match rules are supported by default.
3239 }
3240
3241 //------------------------------------------------------------------------
3242
3243 static inline bool is_pop_count_instr_target(BasicType bt) {
3244 return (is_subword_type(bt) && VM_Version::supports_avx512_bitalg()) ||
3245 (is_non_subword_integral_type(bt) && VM_Version::supports_avx512_vpopcntdq());
3246 }
3247
3248 bool Matcher::match_rule_supported_auto_vectorization(int opcode, int vlen, BasicType bt) {
3249 return match_rule_supported_vector(opcode, vlen, bt);
3250 }
3251
3252 // Identify extra cases that we might want to provide match rules for vector nodes and
3253 // other intrinsics guarded with vector length (vlen) and element type (bt).
3254 bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
3255 if (!match_rule_supported(opcode)) {
3256 return false;
3257 }
3258 // Matcher::vector_size_supported() restricts vector sizes in the following way (see Matcher::vector_width_in_bytes):
3259 // * SSE2 supports 128bit vectors for all types;
3260 // * AVX1 supports 256bit vectors only for FLOAT and DOUBLE types;
3261 // * AVX2 supports 256bit vectors for all types;
3262 // * AVX512F supports 512bit vectors only for INT, FLOAT, and DOUBLE types;
3263 // * AVX512BW supports 512bit vectors for BYTE, SHORT, and CHAR types.
3264 // There's also a limit on minimum vector size supported: 2 elements (or 4 bytes for BYTE).
3265 // And MaxVectorSize is taken into account as well.
3266 if (!vector_size_supported(bt, vlen)) {
3267 return false;
3268 }
3269 // Special cases which require vector length follow:
3270 // * implementation limitations
3271 // * some 512bit vector operations on FLOAT and DOUBLE types require AVX512DQ
3272 // * 128bit vroundpd instruction is present only in AVX1
3273 int size_in_bits = vlen * type2aelembytes(bt) * BitsPerByte;
3274 switch (opcode) {
3275 case Op_MaxVHF:
3276 case Op_MinVHF:
3277 if (!VM_Version::supports_avx512bw()) {
3278 return false;
3279 }
3280 case Op_AddVHF:
3281 case Op_DivVHF:
3282 case Op_FmaVHF:
3283 case Op_MulVHF:
3284 case Op_SubVHF:
3285 case Op_SqrtVHF:
3286 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3287 return false;
3288 }
3289 if (!VM_Version::supports_avx512_fp16()) {
3290 return false;
3291 }
3292 break;
3293 case Op_AbsVF:
3294 case Op_NegVF:
3295 if ((vlen == 16) && (VM_Version::supports_avx512dq() == false)) {
3296 return false; // 512bit vandps and vxorps are not available
3297 }
3298 break;
3299 case Op_AbsVD:
3300 case Op_NegVD:
3301 if ((vlen == 8) && (VM_Version::supports_avx512dq() == false)) {
3302 return false; // 512bit vpmullq, vandpd and vxorpd are not available
3303 }
3304 break;
3305 case Op_RotateRightV:
3306 case Op_RotateLeftV:
3307 if (bt != T_INT && bt != T_LONG) {
3308 return false;
3309 } // fallthrough
3310 case Op_MacroLogicV:
3311 if (!VM_Version::supports_evex() ||
3312 ((size_in_bits != 512) && !VM_Version::supports_avx512vl())) {
3313 return false;
3314 }
3315 break;
3316 case Op_ClearArray:
3317 case Op_VectorMaskGen:
3318 case Op_VectorCmpMasked:
3319 if (!VM_Version::supports_avx512bw()) {
3320 return false;
3321 }
3322 if ((size_in_bits != 512) && !VM_Version::supports_avx512vl()) {
3323 return false;
3324 }
3325 break;
3326 case Op_LoadVectorMasked:
3327 case Op_StoreVectorMasked:
3328 if (!VM_Version::supports_avx512bw() && (is_subword_type(bt) || UseAVX < 1)) {
3329 return false;
3330 }
3331 break;
3332 case Op_UMinV:
3333 case Op_UMaxV:
3334 if (UseAVX == 0) {
3335 return false;
3336 }
3337 break;
3338 case Op_UMinReductionV:
3339 case Op_UMaxReductionV:
3340 if (UseAVX == 0) {
3341 return false;
3342 }
3343 if (bt == T_LONG && !VM_Version::supports_avx512vl()) {
3344 return false;
3345 }
3346 if (UseAVX > 2 && size_in_bits == 512 && !VM_Version::supports_avx512vl()) {
3347 return false;
3348 }
3349 break;
3350 case Op_MaxV:
3351 case Op_MinV:
3352 if (UseSSE < 4 && is_integral_type(bt)) {
3353 return false;
3354 }
3355 if ((bt == T_FLOAT || bt == T_DOUBLE)) {
3356 // Float/Double intrinsics are enabled for AVX family currently.
3357 if (UseAVX == 0) {
3358 return false;
3359 }
3360 if (UseAVX > 2 && (!VM_Version::supports_avx512dq() && size_in_bits == 512)) { // 512 bit Float/Double intrinsics need AVX512DQ
3361 return false;
3362 }
3363 }
3364 break;
3365 case Op_CallLeafVector:
3366 if (size_in_bits == 512 && !VM_Version::supports_avx512vlbwdq()) {
3367 return false;
3368 }
3369 break;
3370 case Op_AddReductionVI:
3371 if (bt == T_INT && (UseSSE < 3 || !VM_Version::supports_ssse3())) {
3372 return false;
3373 }
3374 // fallthrough
3375 case Op_AndReductionV:
3376 case Op_OrReductionV:
3377 case Op_XorReductionV:
3378 if (is_subword_type(bt) && (UseSSE < 4)) {
3379 return false;
3380 }
3381 break;
3382 case Op_MinReductionV:
3383 case Op_MaxReductionV:
3384 if ((bt == T_INT || is_subword_type(bt)) && UseSSE < 4) {
3385 return false;
3386 } else if (bt == T_LONG && (UseAVX < 3 || !VM_Version::supports_avx512vlbwdq())) {
3387 return false;
3388 }
3389 // Float/Double intrinsics enabled for AVX family.
3390 if (UseAVX == 0 && (bt == T_FLOAT || bt == T_DOUBLE)) {
3391 return false;
3392 }
3393 if (UseAVX > 2 && (!VM_Version::supports_avx512dq() && size_in_bits == 512)) {
3394 return false;
3395 }
3396 break;
3397 case Op_VectorBlend:
3398 if (UseAVX == 0 && size_in_bits < 128) {
3399 return false;
3400 }
3401 break;
3402 case Op_VectorTest:
3403 if (UseSSE < 4) {
3404 return false; // Implementation limitation
3405 } else if (size_in_bits < 32) {
3406 return false; // Implementation limitation
3407 }
3408 break;
3409 case Op_VectorLoadShuffle:
3410 case Op_VectorRearrange:
3411 if(vlen == 2) {
3412 return false; // Implementation limitation due to how shuffle is loaded
3413 } else if (size_in_bits == 256 && UseAVX < 2) {
3414 return false; // Implementation limitation
3415 }
3416 break;
3417 case Op_VectorLoadMask:
3418 case Op_VectorMaskCast:
3419 if (size_in_bits == 256 && UseAVX < 2) {
3420 return false; // Implementation limitation
3421 }
3422 // fallthrough
3423 case Op_VectorStoreMask:
3424 if (vlen == 2) {
3425 return false; // Implementation limitation
3426 }
3427 break;
3428 case Op_PopulateIndex:
3429 if (size_in_bits > 256 && !VM_Version::supports_avx512bw()) {
3430 return false;
3431 }
3432 break;
3433 case Op_VectorCastB2X:
3434 case Op_VectorCastS2X:
3435 case Op_VectorCastI2X:
3436 if (bt != T_DOUBLE && size_in_bits == 256 && UseAVX < 2) {
3437 return false;
3438 }
3439 break;
3440 case Op_VectorCastL2X:
3441 if (is_integral_type(bt) && size_in_bits == 256 && UseAVX < 2) {
3442 return false;
3443 } else if (!is_integral_type(bt) && !VM_Version::supports_avx512dq()) {
3444 return false;
3445 }
3446 break;
3447 case Op_VectorCastF2X: {
3448 // As per JLS section 5.1.3 narrowing conversion to sub-word types
3449 // happen after intermediate conversion to integer and special handling
3450 // code needs AVX2 vpcmpeqd instruction for 256 bit vectors.
3451 int src_size_in_bits = type2aelembytes(T_FLOAT) * vlen * BitsPerByte;
3452 if (is_integral_type(bt) && src_size_in_bits == 256 && UseAVX < 2) {
3453 return false;
3454 }
3455 }
3456 // fallthrough
3457 case Op_VectorCastD2X:
3458 if (bt == T_LONG && !VM_Version::supports_avx512dq()) {
3459 return false;
3460 }
3461 break;
3462 case Op_VectorCastF2HF:
3463 case Op_VectorCastHF2F:
3464 if (!VM_Version::supports_f16c() &&
3465 ((!VM_Version::supports_evex() ||
3466 ((size_in_bits != 512) && !VM_Version::supports_avx512vl())))) {
3467 return false;
3468 }
3469 break;
3470 case Op_RoundVD:
3471 if (!VM_Version::supports_avx512dq()) {
3472 return false;
3473 }
3474 break;
3475 case Op_MulReductionVI:
3476 if (bt == T_BYTE && size_in_bits == 512 && !VM_Version::supports_avx512bw()) {
3477 return false;
3478 }
3479 break;
3480 case Op_LoadVectorGatherMasked:
3481 if (!is_subword_type(bt) && size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3482 return false;
3483 }
3484 if (is_subword_type(bt) &&
3485 ((size_in_bits > 256 && !VM_Version::supports_avx512bw()) ||
3486 (size_in_bits < 64) ||
3487 (bt == T_SHORT && !VM_Version::supports_bmi2()))) {
3488 return false;
3489 }
3490 break;
3491 case Op_StoreVectorScatterMasked:
3492 case Op_StoreVectorScatter:
3493 if (is_subword_type(bt)) {
3494 return false;
3495 } else if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3496 return false;
3497 }
3498 // fallthrough
3499 case Op_LoadVectorGather:
3500 if (!is_subword_type(bt) && size_in_bits == 64) {
3501 return false;
3502 }
3503 if (is_subword_type(bt) && size_in_bits < 64) {
3504 return false;
3505 }
3506 break;
3507 case Op_SaturatingAddV:
3508 case Op_SaturatingSubV:
3509 if (UseAVX < 1) {
3510 return false; // Implementation limitation
3511 }
3512 if (is_subword_type(bt) && size_in_bits == 512 && !VM_Version::supports_avx512bw()) {
3513 return false;
3514 }
3515 break;
3516 case Op_SelectFromTwoVector:
3517 if (size_in_bits < 128) {
3518 return false;
3519 }
3520 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3521 return false;
3522 }
3523 if (bt == T_SHORT && !VM_Version::supports_avx512bw()) {
3524 return false;
3525 }
3526 if (bt == T_BYTE && !VM_Version::supports_avx512_vbmi()) {
3527 return false;
3528 }
3529 if ((bt == T_INT || bt == T_FLOAT || bt == T_DOUBLE) && !VM_Version::supports_evex()) {
3530 return false;
3531 }
3532 break;
3533 case Op_MaskAll:
3534 if (!VM_Version::supports_evex()) {
3535 return false;
3536 }
3537 if ((vlen > 16 || is_subword_type(bt)) && !VM_Version::supports_avx512bw()) {
3538 return false;
3539 }
3540 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3541 return false;
3542 }
3543 break;
3544 case Op_VectorMaskCmp:
3545 if (vlen < 2 || size_in_bits < 32) {
3546 return false;
3547 }
3548 break;
3549 case Op_CompressM:
3550 if (UseAVX < 3 || !VM_Version::supports_bmi2()) {
3551 return false;
3552 }
3553 break;
3554 case Op_CompressV:
3555 case Op_ExpandV:
3556 if (is_subword_type(bt) && !VM_Version::supports_avx512_vbmi2()) {
3557 return false;
3558 }
3559 if (size_in_bits < 128 ) {
3560 return false;
3561 }
3562 case Op_VectorLongToMask:
3563 if (UseAVX < 1) {
3564 return false;
3565 }
3566 if (UseAVX < 3 && !VM_Version::supports_bmi2()) {
3567 return false;
3568 }
3569 break;
3570 case Op_SignumVD:
3571 case Op_SignumVF:
3572 if (UseAVX < 1) {
3573 return false;
3574 }
3575 break;
3576 case Op_PopCountVI:
3577 case Op_PopCountVL: {
3578 if (!is_pop_count_instr_target(bt) &&
3579 (size_in_bits == 512) && !VM_Version::supports_avx512bw()) {
3580 return false;
3581 }
3582 }
3583 break;
3584 case Op_ReverseV:
3585 case Op_ReverseBytesV:
3586 if (UseAVX < 2) {
3587 return false;
3588 }
3589 break;
3590 case Op_CountTrailingZerosV:
3591 case Op_CountLeadingZerosV:
3592 if (UseAVX < 2) {
3593 return false;
3594 }
3595 break;
3596 }
3597 return true; // Per default match rules are supported.
3598 }
3599
3600 bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
3601 // ADLC based match_rule_supported routine checks for the existence of pattern based
3602 // on IR opcode. Most of the unary/binary/ternary masked operation share the IR nodes
3603 // of their non-masked counterpart with mask edge being the differentiator.
3604 // This routine does a strict check on the existence of masked operation patterns
3605 // by returning a default false value for all the other opcodes apart from the
3606 // ones whose masked instruction patterns are defined in this file.
3607 if (!match_rule_supported_vector(opcode, vlen, bt)) {
3608 return false;
3609 }
3610
3611 int size_in_bits = vlen * type2aelembytes(bt) * BitsPerByte;
3612 if (size_in_bits != 512 && !VM_Version::supports_avx512vl()) {
3613 return false;
3614 }
3615 switch(opcode) {
3616 // Unary masked operations
3617 case Op_AbsVB:
3618 case Op_AbsVS:
3619 if(!VM_Version::supports_avx512bw()) {
3620 return false; // Implementation limitation
3621 }
3622 case Op_AbsVI:
3623 case Op_AbsVL:
3624 return true;
3625
3626 // Ternary masked operations
3627 case Op_FmaVF:
3628 case Op_FmaVD:
3629 return true;
3630
3631 case Op_MacroLogicV:
3632 if(bt != T_INT && bt != T_LONG) {
3633 return false;
3634 }
3635 return true;
3636
3637 // Binary masked operations
3638 case Op_AddVB:
3639 case Op_AddVS:
3640 case Op_SubVB:
3641 case Op_SubVS:
3642 case Op_MulVS:
3643 case Op_LShiftVS:
3644 case Op_RShiftVS:
3645 case Op_URShiftVS:
3646 assert(size_in_bits == 512 || VM_Version::supports_avx512vl(), "");
3647 if (!VM_Version::supports_avx512bw()) {
3648 return false; // Implementation limitation
3649 }
3650 return true;
3651
3652 case Op_MulVL:
3653 assert(size_in_bits == 512 || VM_Version::supports_avx512vl(), "");
3654 if (!VM_Version::supports_avx512dq()) {
3655 return false; // Implementation limitation
3656 }
3657 return true;
3658
3659 case Op_AndV:
3660 case Op_OrV:
3661 case Op_XorV:
3662 case Op_RotateRightV:
3663 case Op_RotateLeftV:
3664 if (bt != T_INT && bt != T_LONG) {
3665 return false; // Implementation limitation
3666 }
3667 return true;
3668
3669 case Op_VectorLoadMask:
3670 assert(size_in_bits == 512 || VM_Version::supports_avx512vl(), "");
3671 if (is_subword_type(bt) && !VM_Version::supports_avx512bw()) {
3672 return false;
3673 }
3674 return true;
3675
3676 case Op_AddVI:
3677 case Op_AddVL:
3678 case Op_AddVF:
3679 case Op_AddVD:
3680 case Op_SubVI:
3681 case Op_SubVL:
3682 case Op_SubVF:
3683 case Op_SubVD:
3684 case Op_MulVI:
3685 case Op_MulVF:
3686 case Op_MulVD:
3687 case Op_DivVF:
3688 case Op_DivVD:
3689 case Op_SqrtVF:
3690 case Op_SqrtVD:
3691 case Op_LShiftVI:
3692 case Op_LShiftVL:
3693 case Op_RShiftVI:
3694 case Op_RShiftVL:
3695 case Op_URShiftVI:
3696 case Op_URShiftVL:
3697 case Op_LoadVectorMasked:
3698 case Op_StoreVectorMasked:
3699 case Op_LoadVectorGatherMasked:
3700 case Op_StoreVectorScatterMasked:
3701 return true;
3702
3703 case Op_UMinV:
3704 case Op_UMaxV:
3705 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3706 return false;
3707 } // fallthrough
3708 case Op_MaxV:
3709 case Op_MinV:
3710 if (is_subword_type(bt) && !VM_Version::supports_avx512bw()) {
3711 return false; // Implementation limitation
3712 }
3713 if (is_floating_point_type(bt) && !VM_Version::supports_avx10_2()) {
3714 return false; // Implementation limitation
3715 }
3716 return true;
3717 case Op_SaturatingAddV:
3718 case Op_SaturatingSubV:
3719 if (!is_subword_type(bt)) {
3720 return false;
3721 }
3722 if (size_in_bits < 128 || !VM_Version::supports_avx512bw()) {
3723 return false; // Implementation limitation
3724 }
3725 return true;
3726
3727 case Op_VectorMaskCmp:
3728 if (is_subword_type(bt) && !VM_Version::supports_avx512bw()) {
3729 return false; // Implementation limitation
3730 }
3731 return true;
3732
3733 case Op_VectorRearrange:
3734 if (bt == T_SHORT && !VM_Version::supports_avx512bw()) {
3735 return false; // Implementation limitation
3736 }
3737 if (bt == T_BYTE && !VM_Version::supports_avx512_vbmi()) {
3738 return false; // Implementation limitation
3739 } else if ((bt == T_INT || bt == T_FLOAT) && size_in_bits < 256) {
3740 return false; // Implementation limitation
3741 }
3742 return true;
3743
3744 // Binary Logical operations
3745 case Op_AndVMask:
3746 case Op_OrVMask:
3747 case Op_XorVMask:
3748 if (vlen > 16 && !VM_Version::supports_avx512bw()) {
3749 return false; // Implementation limitation
3750 }
3751 return true;
3752
3753 case Op_PopCountVI:
3754 case Op_PopCountVL:
3755 if (!is_pop_count_instr_target(bt)) {
3756 return false;
3757 }
3758 return true;
3759
3760 case Op_MaskAll:
3761 return true;
3762
3763 case Op_CountLeadingZerosV:
3764 if (is_non_subword_integral_type(bt) && VM_Version::supports_avx512cd()) {
3765 return true;
3766 }
3767 default:
3768 return false;
3769 }
3770 }
3771
3772 bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* vt) {
3773 return false;
3774 }
3775
3776 // Return true if Vector::rearrange needs preparation of the shuffle argument
3777 bool Matcher::vector_rearrange_requires_load_shuffle(BasicType elem_bt, int vlen) {
3778 switch (elem_bt) {
3779 case T_BYTE: return false;
3780 case T_SHORT: return !VM_Version::supports_avx512bw();
3781 case T_INT: return !VM_Version::supports_avx();
3782 case T_LONG: return vlen < 8 && !VM_Version::supports_avx512vl();
3783 default:
3784 ShouldNotReachHere();
3785 return false;
3786 }
3787 }
3788
3789 bool Matcher::mask_op_prefers_predicate(int opcode, const TypeVect* vt) {
3790 // Prefer predicate if the mask type is "TypePVectMask".
3791 return vt->isa_pvectmask() != nullptr;
3792 }
3793
3794 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
3795 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
3796 bool legacy = (generic_opnd->opcode() == LEGVEC);
3797 if (!VM_Version::supports_avx512vlbwdq() && // KNL
3798 is_temp && !legacy && (ideal_reg == Op_VecZ)) {
3799 // Conservatively specialize 512bit vec TEMP operands to legVecZ (zmm0-15) on KNL.
3800 return new legVecZOper();
3801 }
3802 if (legacy) {
3803 switch (ideal_reg) {
3804 case Op_VecS: return new legVecSOper();
3805 case Op_VecD: return new legVecDOper();
3806 case Op_VecX: return new legVecXOper();
3807 case Op_VecY: return new legVecYOper();
3808 case Op_VecZ: return new legVecZOper();
3809 }
3810 } else {
3811 switch (ideal_reg) {
3812 case Op_VecS: return new vecSOper();
3813 case Op_VecD: return new vecDOper();
3814 case Op_VecX: return new vecXOper();
3815 case Op_VecY: return new vecYOper();
3816 case Op_VecZ: return new vecZOper();
3817 }
3818 }
3819 ShouldNotReachHere();
3820 return nullptr;
3821 }
3822
3823 bool Matcher::is_reg2reg_move(MachNode* m) {
3824 switch (m->rule()) {
3825 case MoveVec2Leg_rule:
3826 case MoveLeg2Vec_rule:
3827 case MoveF2VL_rule:
3828 case MoveF2LEG_rule:
3829 case MoveVL2F_rule:
3830 case MoveLEG2F_rule:
3831 case MoveD2VL_rule:
3832 case MoveD2LEG_rule:
3833 case MoveVL2D_rule:
3834 case MoveLEG2D_rule:
3835 return true;
3836 default:
3837 return false;
3838 }
3839 }
3840
3841 bool Matcher::is_generic_vector(MachOper* opnd) {
3842 switch (opnd->opcode()) {
3843 case VEC:
3844 case LEGVEC:
3845 return true;
3846 default:
3847 return false;
3848 }
3849 }
3850
3851 //------------------------------------------------------------------------
3852
3853 const RegMask* Matcher::predicate_reg_mask(void) {
3854 return &_VECTMASK_REG_mask;
3855 }
3856
3857 // Max vector size in bytes. 0 if not supported.
3858 int Matcher::vector_width_in_bytes(BasicType bt) {
3859 assert(is_java_primitive(bt), "only primitive type vectors");
3860 // SSE2 supports 128bit vectors for all types.
3861 // AVX2 supports 256bit vectors for all types.
3862 // AVX2/EVEX supports 512bit vectors for all types.
3863 int size = (UseAVX > 1) ? (1 << UseAVX) * 8 : 16;
3864 // AVX1 supports 256bit vectors only for FLOAT and DOUBLE.
3865 if (UseAVX > 0 && (bt == T_FLOAT || bt == T_DOUBLE))
3866 size = (UseAVX > 2) ? 64 : 32;
3867 if (UseAVX > 2 && (bt == T_BYTE || bt == T_SHORT || bt == T_CHAR))
3868 size = (VM_Version::supports_avx512bw()) ? 64 : 32;
3869 // Use flag to limit vector size.
3870 size = MIN2(size,(int)MaxVectorSize);
3871 // Minimum 2 values in vector (or 4 for bytes).
3872 switch (bt) {
3873 case T_DOUBLE:
3874 case T_LONG:
3875 if (size < 16) return 0;
3876 break;
3877 case T_FLOAT:
3878 case T_INT:
3879 if (size < 8) return 0;
3880 break;
3881 case T_BOOLEAN:
3882 if (size < 4) return 0;
3883 break;
3884 case T_CHAR:
3885 if (size < 4) return 0;
3886 break;
3887 case T_BYTE:
3888 if (size < 4) return 0;
3889 break;
3890 case T_SHORT:
3891 if (size < 4) return 0;
3892 break;
3893 default:
3894 ShouldNotReachHere();
3895 }
3896 return size;
3897 }
3898
3899 // Limits on vector size (number of elements) loaded into vector.
3900 int Matcher::max_vector_size(const BasicType bt) {
3901 return vector_width_in_bytes(bt)/type2aelembytes(bt);
3902 }
3903 int Matcher::min_vector_size(const BasicType bt) {
3904 int max_size = max_vector_size(bt);
3905 // Min size which can be loaded into vector is 4 bytes.
3906 int size = (type2aelembytes(bt) == 1) ? 4 : 2;
3907 // Support for calling svml double64 vectors
3908 if (bt == T_DOUBLE) {
3909 size = 1;
3910 }
3911 return MIN2(size,max_size);
3912 }
3913
3914 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
3915 // Limit the max vector size for auto vectorization to 256 bits (32 bytes)
3916 // by default on Cascade Lake
3917 if (VM_Version::is_default_intel_cascade_lake()) {
3918 return MIN2(Matcher::max_vector_size(bt), 32 / type2aelembytes(bt));
3919 }
3920 return Matcher::max_vector_size(bt);
3921 }
3922
3923 int Matcher::scalable_vector_reg_size(const BasicType bt) {
3924 return -1;
3925 }
3926
3927 // Vector ideal reg corresponding to specified size in bytes
3928 uint Matcher::vector_ideal_reg(int size) {
3929 assert(MaxVectorSize >= size, "");
3930 switch(size) {
3931 case 4: return Op_VecS;
3932 case 8: return Op_VecD;
3933 case 16: return Op_VecX;
3934 case 32: return Op_VecY;
3935 case 64: return Op_VecZ;
3936 }
3937 ShouldNotReachHere();
3938 return 0;
3939 }
3940
3941 // Check for shift by small constant as well
3942 static bool clone_shift(Node* shift, Matcher* matcher, Matcher::MStack& mstack, VectorSet& address_visited) {
3943 if (shift->Opcode() == Op_LShiftX && shift->in(2)->is_Con() &&
3944 shift->in(2)->get_int() <= 3 &&
3945 // Are there other uses besides address expressions?
3946 !matcher->is_visited(shift)) {
3947 address_visited.set(shift->_idx); // Flag as address_visited
3948 mstack.push(shift->in(2), Matcher::Visit);
3949 Node *conv = shift->in(1);
3950 // Allow Matcher to match the rule which bypass
3951 // ConvI2L operation for an array index on LP64
3952 // if the index value is positive.
3953 if (conv->Opcode() == Op_ConvI2L &&
3954 conv->as_Type()->type()->is_long()->_lo >= 0 &&
3955 // Are there other uses besides address expressions?
3956 !matcher->is_visited(conv)) {
3957 address_visited.set(conv->_idx); // Flag as address_visited
3958 mstack.push(conv->in(1), Matcher::Pre_Visit);
3959 } else {
3960 mstack.push(conv, Matcher::Pre_Visit);
3961 }
3962 return true;
3963 }
3964 return false;
3965 }
3966
3967 // This function identifies sub-graphs in which a 'load' node is
3968 // input to two different nodes, and such that it can be matched
3969 // with BMI instructions like blsi, blsr, etc.
3970 // Example : for b = -a[i] & a[i] can be matched to blsi r32, m32.
3971 // The graph is (AndL (SubL Con0 LoadL*) LoadL*), where LoadL*
3972 // refers to the same node.
3973 //
3974 // Match the generic fused operations pattern (op1 (op2 Con{ConType} mop) mop)
3975 // This is a temporary solution until we make DAGs expressible in ADL.
3976 template<typename ConType>
3977 class FusedPatternMatcher {
3978 Node* _op1_node;
3979 Node* _mop_node;
3980 int _con_op;
3981
3982 static int match_next(Node* n, int next_op, int next_op_idx) {
3983 if (n->in(1) == nullptr || n->in(2) == nullptr) {
3984 return -1;
3985 }
3986
3987 if (next_op_idx == -1) { // n is commutative, try rotations
3988 if (n->in(1)->Opcode() == next_op) {
3989 return 1;
3990 } else if (n->in(2)->Opcode() == next_op) {
3991 return 2;
3992 }
3993 } else {
3994 assert(next_op_idx > 0 && next_op_idx <= 2, "Bad argument index");
3995 if (n->in(next_op_idx)->Opcode() == next_op) {
3996 return next_op_idx;
3997 }
3998 }
3999 return -1;
4000 }
4001
4002 public:
4003 FusedPatternMatcher(Node* op1_node, Node* mop_node, int con_op) :
4004 _op1_node(op1_node), _mop_node(mop_node), _con_op(con_op) { }
4005
4006 bool match(int op1, int op1_op2_idx, // op1 and the index of the op1->op2 edge, -1 if op1 is commutative
4007 int op2, int op2_con_idx, // op2 and the index of the op2->con edge, -1 if op2 is commutative
4008 typename ConType::NativeType con_value) {
4009 if (_op1_node->Opcode() != op1) {
4010 return false;
4011 }
4012 if (_mop_node->outcnt() > 2) {
4013 return false;
4014 }
4015 op1_op2_idx = match_next(_op1_node, op2, op1_op2_idx);
4016 if (op1_op2_idx == -1) {
4017 return false;
4018 }
4019 // Memory operation must be the other edge
4020 int op1_mop_idx = (op1_op2_idx & 1) + 1;
4021
4022 // Check that the mop node is really what we want
4023 if (_op1_node->in(op1_mop_idx) == _mop_node) {
4024 Node* op2_node = _op1_node->in(op1_op2_idx);
4025 if (op2_node->outcnt() > 1) {
4026 return false;
4027 }
4028 assert(op2_node->Opcode() == op2, "Should be");
4029 op2_con_idx = match_next(op2_node, _con_op, op2_con_idx);
4030 if (op2_con_idx == -1) {
4031 return false;
4032 }
4033 // Memory operation must be the other edge
4034 int op2_mop_idx = (op2_con_idx & 1) + 1;
4035 // Check that the memory operation is the same node
4036 if (op2_node->in(op2_mop_idx) == _mop_node) {
4037 // Now check the constant
4038 const Type* con_type = op2_node->in(op2_con_idx)->bottom_type();
4039 if (con_type != Type::TOP && ConType::as_self(con_type)->get_con() == con_value) {
4040 return true;
4041 }
4042 }
4043 }
4044 return false;
4045 }
4046 };
4047
4048 static bool is_bmi_pattern(Node* n, Node* m) {
4049 assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "sanity");
4050 if (n != nullptr && m != nullptr) {
4051 if (m->Opcode() == Op_LoadI) {
4052 FusedPatternMatcher<TypeInt> bmii(n, m, Op_ConI);
4053 return bmii.match(Op_AndI, -1, Op_SubI, 1, 0) ||
4054 bmii.match(Op_AndI, -1, Op_AddI, -1, -1) ||
4055 bmii.match(Op_XorI, -1, Op_AddI, -1, -1);
4056 } else if (m->Opcode() == Op_LoadL) {
4057 FusedPatternMatcher<TypeLong> bmil(n, m, Op_ConL);
4058 return bmil.match(Op_AndL, -1, Op_SubL, 1, 0) ||
4059 bmil.match(Op_AndL, -1, Op_AddL, -1, -1) ||
4060 bmil.match(Op_XorL, -1, Op_AddL, -1, -1);
4061 }
4062 }
4063 return false;
4064 }
4065
4066 // Should the matcher clone input 'm' of node 'n'?
4067 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
4068 // If 'n' and 'm' are part of a graph for BMI instruction, clone the input 'm'.
4069 if (VM_Version::supports_bmi1() && VM_Version::supports_avx() && is_bmi_pattern(n, m)) {
4070 mstack.push(m, Visit);
4071 return true;
4072 }
4073 if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
4074 mstack.push(m, Visit); // m = ShiftCntV
4075 return true;
4076 }
4077 if (is_encode_and_store_pattern(n, m)) {
4078 mstack.push(m, Visit);
4079 return true;
4080 }
4081 return false;
4082 }
4083
4084 // Should the Matcher clone shifts on addressing modes, expecting them
4085 // to be subsumed into complex addressing expressions or compute them
4086 // into registers?
4087 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
4088 Node *off = m->in(AddPNode::Offset);
4089 if (off->is_Con()) {
4090 address_visited.test_set(m->_idx); // Flag as address_visited
4091 Node *adr = m->in(AddPNode::Address);
4092
4093 // Intel can handle 2 adds in addressing mode, with one of them using an immediate offset.
4094 // AtomicAdd is not an addressing expression.
4095 // Cheap to find it by looking for screwy base.
4096 if (adr->is_AddP() &&
4097 !adr->in(AddPNode::Base)->is_top() &&
4098 !adr->in(AddPNode::Offset)->is_Con() &&
4099 off->get_long() == (int) (off->get_long()) && // immL32
4100 // Are there other uses besides address expressions?
4101 !is_visited(adr)) {
4102 address_visited.set(adr->_idx); // Flag as address_visited
4103 Node *shift = adr->in(AddPNode::Offset);
4104 if (!clone_shift(shift, this, mstack, address_visited)) {
4105 mstack.push(shift, Pre_Visit);
4106 }
4107 mstack.push(adr->in(AddPNode::Address), Pre_Visit);
4108 mstack.push(adr->in(AddPNode::Base), Pre_Visit);
4109 } else {
4110 mstack.push(adr, Pre_Visit);
4111 }
4112
4113 // Clone X+offset as it also folds into most addressing expressions
4114 mstack.push(off, Visit);
4115 mstack.push(m->in(AddPNode::Base), Pre_Visit);
4116 return true;
4117 } else if (clone_shift(off, this, mstack, address_visited)) {
4118 address_visited.test_set(m->_idx); // Flag as address_visited
4119 mstack.push(m->in(AddPNode::Address), Pre_Visit);
4120 mstack.push(m->in(AddPNode::Base), Pre_Visit);
4121 return true;
4122 }
4123 return false;
4124 }
4125
4126 static inline Assembler::ComparisonPredicate booltest_pred_to_comparison_pred(int bt) {
4127 switch (bt) {
4128 case BoolTest::eq:
4129 return Assembler::eq;
4130 case BoolTest::ne:
4131 return Assembler::neq;
4132 case BoolTest::le:
4133 case BoolTest::ule:
4134 return Assembler::le;
4135 case BoolTest::ge:
4136 case BoolTest::uge:
4137 return Assembler::nlt;
4138 case BoolTest::lt:
4139 case BoolTest::ult:
4140 return Assembler::lt;
4141 case BoolTest::gt:
4142 case BoolTest::ugt:
4143 return Assembler::nle;
4144 default : ShouldNotReachHere(); return Assembler::_false;
4145 }
4146 }
4147
4148 static inline Assembler::ComparisonPredicateFP booltest_pred_to_comparison_pred_fp(int bt) {
4149 switch (bt) {
4150 case BoolTest::eq: return Assembler::EQ_OQ; // ordered non-signaling
4151 // As per JLS 15.21.1, != of NaNs is true. Thus use unordered compare.
4152 case BoolTest::ne: return Assembler::NEQ_UQ; // unordered non-signaling
4153 case BoolTest::le: return Assembler::LE_OQ; // ordered non-signaling
4154 case BoolTest::ge: return Assembler::GE_OQ; // ordered non-signaling
4155 case BoolTest::lt: return Assembler::LT_OQ; // ordered non-signaling
4156 case BoolTest::gt: return Assembler::GT_OQ; // ordered non-signaling
4157 default: ShouldNotReachHere(); return Assembler::FALSE_OS;
4158 }
4159 }
4160
4161 // Helper methods for MachSpillCopyNode::implementation().
4162 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
4163 int src_hi, int dst_hi, uint ireg, outputStream* st) {
4164 assert(ireg == Op_VecS || // 32bit vector
4165 ((src_lo & 1) == 0 && (src_lo + 1) == src_hi &&
4166 (dst_lo & 1) == 0 && (dst_lo + 1) == dst_hi),
4167 "no non-adjacent vector moves" );
4168 if (masm) {
4169 switch (ireg) {
4170 case Op_VecS: // copy whole register
4171 case Op_VecD:
4172 case Op_VecX:
4173 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4174 __ movdqu(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]));
4175 } else {
4176 __ vextractf32x4(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]), 0x0);
4177 }
4178 break;
4179 case Op_VecY:
4180 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4181 __ vmovdqu(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]));
4182 } else {
4183 __ vextractf64x4(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]), 0x0);
4184 }
4185 break;
4186 case Op_VecZ:
4187 __ evmovdquq(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]), 2);
4188 break;
4189 default:
4190 ShouldNotReachHere();
4191 }
4192 #ifndef PRODUCT
4193 } else {
4194 switch (ireg) {
4195 case Op_VecS:
4196 case Op_VecD:
4197 case Op_VecX:
4198 st->print("movdqu %s,%s\t# spill",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
4199 break;
4200 case Op_VecY:
4201 case Op_VecZ:
4202 st->print("vmovdqu %s,%s\t# spill",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
4203 break;
4204 default:
4205 ShouldNotReachHere();
4206 }
4207 #endif
4208 }
4209 }
4210
4211 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
4212 int stack_offset, int reg, uint ireg, outputStream* st) {
4213 if (masm) {
4214 if (is_load) {
4215 switch (ireg) {
4216 case Op_VecS:
4217 __ movdl(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4218 break;
4219 case Op_VecD:
4220 __ movq(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4221 break;
4222 case Op_VecX:
4223 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4224 __ movdqu(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4225 } else {
4226 __ vpxor(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), 2);
4227 __ vinsertf32x4(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset),0x0);
4228 }
4229 break;
4230 case Op_VecY:
4231 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4232 __ vmovdqu(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4233 } else {
4234 __ vpxor(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), 2);
4235 __ vinsertf64x4(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset),0x0);
4236 }
4237 break;
4238 case Op_VecZ:
4239 __ evmovdquq(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset), 2);
4240 break;
4241 default:
4242 ShouldNotReachHere();
4243 }
4244 } else { // store
4245 switch (ireg) {
4246 case Op_VecS:
4247 __ movdl(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4248 break;
4249 case Op_VecD:
4250 __ movq(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4251 break;
4252 case Op_VecX:
4253 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4254 __ movdqu(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4255 }
4256 else {
4257 __ vextractf32x4(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]), 0x0);
4258 }
4259 break;
4260 case Op_VecY:
4261 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4262 __ vmovdqu(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4263 }
4264 else {
4265 __ vextractf64x4(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]), 0x0);
4266 }
4267 break;
4268 case Op_VecZ:
4269 __ evmovdquq(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]), 2);
4270 break;
4271 default:
4272 ShouldNotReachHere();
4273 }
4274 }
4275 #ifndef PRODUCT
4276 } else {
4277 if (is_load) {
4278 switch (ireg) {
4279 case Op_VecS:
4280 st->print("movd %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4281 break;
4282 case Op_VecD:
4283 st->print("movq %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4284 break;
4285 case Op_VecX:
4286 st->print("movdqu %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4287 break;
4288 case Op_VecY:
4289 case Op_VecZ:
4290 st->print("vmovdqu %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4291 break;
4292 default:
4293 ShouldNotReachHere();
4294 }
4295 } else { // store
4296 switch (ireg) {
4297 case Op_VecS:
4298 st->print("movd [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4299 break;
4300 case Op_VecD:
4301 st->print("movq [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4302 break;
4303 case Op_VecX:
4304 st->print("movdqu [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4305 break;
4306 case Op_VecY:
4307 case Op_VecZ:
4308 st->print("vmovdqu [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4309 break;
4310 default:
4311 ShouldNotReachHere();
4312 }
4313 }
4314 #endif
4315 }
4316 }
4317
4318 template <class T>
4319 static inline GrowableArray<jbyte>* vreplicate_imm(BasicType bt, T con, int len) {
4320 int size = type2aelembytes(bt) * len;
4321 GrowableArray<jbyte>* val = new GrowableArray<jbyte>(size, size, 0);
4322 for (int i = 0; i < len; i++) {
4323 int offset = i * type2aelembytes(bt);
4324 switch (bt) {
4325 case T_BYTE: val->at(i) = con; break;
4326 case T_SHORT: {
4327 jshort c = con;
4328 memcpy(val->adr_at(offset), &c, sizeof(jshort));
4329 break;
4330 }
4331 case T_INT: {
4332 jint c = con;
4333 memcpy(val->adr_at(offset), &c, sizeof(jint));
4334 break;
4335 }
4336 case T_LONG: {
4337 jlong c = con;
4338 memcpy(val->adr_at(offset), &c, sizeof(jlong));
4339 break;
4340 }
4341 case T_FLOAT: {
4342 jfloat c = con;
4343 memcpy(val->adr_at(offset), &c, sizeof(jfloat));
4344 break;
4345 }
4346 case T_DOUBLE: {
4347 jdouble c = con;
4348 memcpy(val->adr_at(offset), &c, sizeof(jdouble));
4349 break;
4350 }
4351 default: assert(false, "%s", type2name(bt));
4352 }
4353 }
4354 return val;
4355 }
4356
4357 static inline jlong high_bit_set(BasicType bt) {
4358 switch (bt) {
4359 case T_BYTE: return 0x8080808080808080;
4360 case T_SHORT: return 0x8000800080008000;
4361 case T_INT: return 0x8000000080000000;
4362 case T_LONG: return 0x8000000000000000;
4363 default:
4364 ShouldNotReachHere();
4365 return 0;
4366 }
4367 }
4368
4369 #ifndef PRODUCT
4370 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
4371 st->print("nop \t# %d bytes pad for loops and calls", _count);
4372 }
4373 #endif
4374
4375 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
4376 __ nop(_count);
4377 }
4378
4379 uint MachNopNode::size(PhaseRegAlloc*) const {
4380 return _count;
4381 }
4382
4383 #ifndef PRODUCT
4384 void MachBreakpointNode::format(PhaseRegAlloc*, outputStream* st) const {
4385 st->print("# breakpoint");
4386 }
4387 #endif
4388
4389 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const {
4390 __ int3();
4391 }
4392
4393 uint MachBreakpointNode::size(PhaseRegAlloc* ra_) const {
4394 return MachNode::size(ra_);
4395 }
4396
4397 %}
4398
4399 //----------ENCODING BLOCK-----------------------------------------------------
4400 // This block specifies the encoding classes used by the compiler to
4401 // output byte streams. Encoding classes are parameterized macros
4402 // used by Machine Instruction Nodes in order to generate the bit
4403 // encoding of the instruction. Operands specify their base encoding
4404 // interface with the interface keyword. There are currently
4405 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
4406 // COND_INTER. REG_INTER causes an operand to generate a function
4407 // which returns its register number when queried. CONST_INTER causes
4408 // an operand to generate a function which returns the value of the
4409 // constant when queried. MEMORY_INTER causes an operand to generate
4410 // four functions which return the Base Register, the Index Register,
4411 // the Scale Value, and the Offset Value of the operand when queried.
4412 // COND_INTER causes an operand to generate six functions which return
4413 // the encoding code (ie - encoding bits for the instruction)
4414 // associated with each basic boolean condition for a conditional
4415 // instruction.
4416 //
4417 // Instructions specify two basic values for encoding. Again, a
4418 // function is available to check if the constant displacement is an
4419 // oop. They use the ins_encode keyword to specify their encoding
4420 // classes (which must be a sequence of enc_class names, and their
4421 // parameters, specified in the encoding block), and they use the
4422 // opcode keyword to specify, in order, their primary, secondary, and
4423 // tertiary opcode. Only the opcode sections which a particular
4424 // instruction needs for encoding need to be specified.
4425 encode %{
4426 enc_class cdql_enc(no_rax_rdx_RegI div)
4427 %{
4428 // Full implementation of Java idiv and irem; checks for
4429 // special case as described in JVM spec., p.243 & p.271.
4430 //
4431 // normal case special case
4432 //
4433 // input : rax: dividend min_int
4434 // reg: divisor -1
4435 //
4436 // output: rax: quotient (= rax idiv reg) min_int
4437 // rdx: remainder (= rax irem reg) 0
4438 //
4439 // Code sequnce:
4440 //
4441 // 0: 3d 00 00 00 80 cmp $0x80000000,%eax
4442 // 5: 75 07/08 jne e <normal>
4443 // 7: 33 d2 xor %edx,%edx
4444 // [div >= 8 -> offset + 1]
4445 // [REX_B]
4446 // 9: 83 f9 ff cmp $0xffffffffffffffff,$div
4447 // c: 74 03/04 je 11 <done>
4448 // 000000000000000e <normal>:
4449 // e: 99 cltd
4450 // [div >= 8 -> offset + 1]
4451 // [REX_B]
4452 // f: f7 f9 idiv $div
4453 // 0000000000000011 <done>:
4454 Label normal;
4455 Label done;
4456
4457 // cmp $0x80000000,%eax
4458 __ cmpl(as_Register(RAX_enc), 0x80000000);
4459
4460 // jne e <normal>
4461 __ jccb(Assembler::notEqual, normal);
4462
4463 // xor %edx,%edx
4464 __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
4465
4466 // cmp $0xffffffffffffffff,%ecx
4467 __ cmpl($div$$Register, -1);
4468
4469 // je 11 <done>
4470 __ jccb(Assembler::equal, done);
4471
4472 // <normal>
4473 // cltd
4474 __ bind(normal);
4475 __ cdql();
4476
4477 // idivl
4478 // <done>
4479 __ idivl($div$$Register);
4480 __ bind(done);
4481 %}
4482
4483 enc_class cdqq_enc(no_rax_rdx_RegL div)
4484 %{
4485 // Full implementation of Java ldiv and lrem; checks for
4486 // special case as described in JVM spec., p.243 & p.271.
4487 //
4488 // normal case special case
4489 //
4490 // input : rax: dividend min_long
4491 // reg: divisor -1
4492 //
4493 // output: rax: quotient (= rax idiv reg) min_long
4494 // rdx: remainder (= rax irem reg) 0
4495 //
4496 // Code sequnce:
4497 //
4498 // 0: 48 ba 00 00 00 00 00 mov $0x8000000000000000,%rdx
4499 // 7: 00 00 80
4500 // a: 48 39 d0 cmp %rdx,%rax
4501 // d: 75 08 jne 17 <normal>
4502 // f: 33 d2 xor %edx,%edx
4503 // 11: 48 83 f9 ff cmp $0xffffffffffffffff,$div
4504 // 15: 74 05 je 1c <done>
4505 // 0000000000000017 <normal>:
4506 // 17: 48 99 cqto
4507 // 19: 48 f7 f9 idiv $div
4508 // 000000000000001c <done>:
4509 Label normal;
4510 Label done;
4511
4512 // mov $0x8000000000000000,%rdx
4513 __ mov64(as_Register(RDX_enc), 0x8000000000000000);
4514
4515 // cmp %rdx,%rax
4516 __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
4517
4518 // jne 17 <normal>
4519 __ jccb(Assembler::notEqual, normal);
4520
4521 // xor %edx,%edx
4522 __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
4523
4524 // cmp $0xffffffffffffffff,$div
4525 __ cmpq($div$$Register, -1);
4526
4527 // je 1e <done>
4528 __ jccb(Assembler::equal, done);
4529
4530 // <normal>
4531 // cqto
4532 __ bind(normal);
4533 __ cdqq();
4534
4535 // idivq (note: must be emitted by the user of this rule)
4536 // <done>
4537 __ idivq($div$$Register);
4538 __ bind(done);
4539 %}
4540
4541 enc_class clear_avx %{
4542 DEBUG_ONLY(int off0 = __ offset());
4543 if (generate_vzeroupper(Compile::current())) {
4544 // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
4545 // Clear upper bits of YMM registers when current compiled code uses
4546 // wide vectors to avoid AVX <-> SSE transition penalty during call.
4547 __ vzeroupper();
4548 }
4549 DEBUG_ONLY(int off1 = __ offset());
4550 assert(off1 - off0 == clear_avx_size(), "correct size prediction");
4551 %}
4552
4553 enc_class Java_To_Runtime(method meth) %{
4554 __ lea(r10, RuntimeAddress((address)$meth$$method));
4555 __ call(r10);
4556 __ post_call_nop();
4557 %}
4558
4559 enc_class Java_Static_Call(method meth)
4560 %{
4561 // JAVA STATIC CALL
4562 // CALL to fixup routine. Fixup routine uses ScopeDesc info to
4563 // determine who we intended to call.
4564 if (!_method) {
4565 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
4566 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
4567 // The NOP here is purely to ensure that eliding a call to
4568 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
4569 __ nop(5);
4570 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
4571 } else {
4572 int method_index = resolved_method_index(masm);
4573 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
4574 : static_call_Relocation::spec(method_index);
4575 address mark = __ pc();
4576 int call_offset = __ offset();
4577 __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
4578 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
4579 // Calls of the same statically bound method can share
4580 // a stub to the interpreter.
4581 __ code()->shared_stub_to_interp_for(_method, call_offset);
4582 } else {
4583 // Emit stubs for static call.
4584 address stub = CompiledDirectCall::emit_to_interp_stub(masm, mark);
4585 __ clear_inst_mark();
4586 if (stub == nullptr) {
4587 ciEnv::current()->record_failure("CodeCache is full");
4588 return;
4589 }
4590 }
4591 }
4592 __ post_call_nop();
4593 %}
4594
4595 enc_class Java_Dynamic_Call(method meth) %{
4596 __ ic_call((address)$meth$$method, resolved_method_index(masm));
4597 __ post_call_nop();
4598 %}
4599
4600 enc_class call_epilog %{
4601 if (VerifyStackAtCalls) {
4602 // Check that stack depth is unchanged: find majik cookie on stack
4603 int framesize = ra_->reg2offset_unchecked(OptoReg::add(ra_->_matcher._old_SP, -3*VMRegImpl::slots_per_word));
4604 Label L;
4605 __ cmpptr(Address(rsp, framesize), (int32_t)0xbadb100d);
4606 __ jccb(Assembler::equal, L);
4607 // Die if stack mismatch
4608 __ int3();
4609 __ bind(L);
4610 }
4611 %}
4612
4613 %}
4614
4615 //----------FRAME--------------------------------------------------------------
4616 // Definition of frame structure and management information.
4617 //
4618 // S T A C K L A Y O U T Allocators stack-slot number
4619 // | (to get allocators register number
4620 // G Owned by | | v add OptoReg::stack0())
4621 // r CALLER | |
4622 // o | +--------+ pad to even-align allocators stack-slot
4623 // w V | pad0 | numbers; owned by CALLER
4624 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
4625 // h ^ | in | 5
4626 // | | args | 4 Holes in incoming args owned by SELF
4627 // | | | | 3
4628 // | | +--------+
4629 // V | | old out| Empty on Intel, window on Sparc
4630 // | old |preserve| Must be even aligned.
4631 // | SP-+--------+----> Matcher::_old_SP, even aligned
4632 // | | in | 3 area for Intel ret address
4633 // Owned by |preserve| Empty on Sparc.
4634 // SELF +--------+
4635 // | | pad2 | 2 pad to align old SP
4636 // | +--------+ 1
4637 // | | locks | 0
4638 // | +--------+----> OptoReg::stack0(), even aligned
4639 // | | pad1 | 11 pad to align new SP
4640 // | +--------+
4641 // | | | 10
4642 // | | spills | 9 spills
4643 // V | | 8 (pad0 slot for callee)
4644 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
4645 // ^ | out | 7
4646 // | | args | 6 Holes in outgoing args owned by CALLEE
4647 // Owned by +--------+
4648 // CALLEE | new out| 6 Empty on Intel, window on Sparc
4649 // | new |preserve| Must be even-aligned.
4650 // | SP-+--------+----> Matcher::_new_SP, even aligned
4651 // | | |
4652 //
4653 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
4654 // known from SELF's arguments and the Java calling convention.
4655 // Region 6-7 is determined per call site.
4656 // Note 2: If the calling convention leaves holes in the incoming argument
4657 // area, those holes are owned by SELF. Holes in the outgoing area
4658 // are owned by the CALLEE. Holes should not be necessary in the
4659 // incoming area, as the Java calling convention is completely under
4660 // the control of the AD file. Doubles can be sorted and packed to
4661 // avoid holes. Holes in the outgoing arguments may be necessary for
4662 // varargs C calling conventions.
4663 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
4664 // even aligned with pad0 as needed.
4665 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
4666 // region 6-11 is even aligned; it may be padded out more so that
4667 // the region from SP to FP meets the minimum stack alignment.
4668 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
4669 // alignment. Region 11, pad1, may be dynamically extended so that
4670 // SP meets the minimum alignment.
4671
4672 frame
4673 %{
4674 // These three registers define part of the calling convention
4675 // between compiled code and the interpreter.
4676 inline_cache_reg(RAX); // Inline Cache Register
4677
4678 // Optional: name the operand used by cisc-spilling to access
4679 // [stack_pointer + offset]
4680 cisc_spilling_operand_name(indOffset32);
4681
4682 // Number of stack slots consumed by locking an object
4683 sync_stack_slots(2);
4684
4685 // Compiled code's Frame Pointer
4686 frame_pointer(RSP);
4687
4688 // Stack alignment requirement
4689 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
4690
4691 // Number of outgoing stack slots killed above the out_preserve_stack_slots
4692 // for calls to C. Supports the var-args backing area for register parms.
4693 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
4694
4695 // The after-PROLOG location of the return address. Location of
4696 // return address specifies a type (REG or STACK) and a number
4697 // representing the register number (i.e. - use a register name) or
4698 // stack slot.
4699 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4700 // Otherwise, it is above the locks and verification slot and alignment word
4701 return_addr(STACK - 2 +
4702 align_up((Compile::current()->in_preserve_stack_slots() +
4703 Compile::current()->fixed_slots()),
4704 stack_alignment_in_slots()));
4705
4706 // Location of compiled Java return values. Same as C for now.
4707 return_value
4708 %{
4709 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4710 "only return normal values");
4711
4712 static const int lo[Op_RegL + 1] = {
4713 0,
4714 0,
4715 RAX_num, // Op_RegN
4716 RAX_num, // Op_RegI
4717 RAX_num, // Op_RegP
4718 XMM0_num, // Op_RegF
4719 XMM0_num, // Op_RegD
4720 RAX_num // Op_RegL
4721 };
4722 static const int hi[Op_RegL + 1] = {
4723 0,
4724 0,
4725 OptoReg::Bad, // Op_RegN
4726 OptoReg::Bad, // Op_RegI
4727 RAX_H_num, // Op_RegP
4728 OptoReg::Bad, // Op_RegF
4729 XMM0b_num, // Op_RegD
4730 RAX_H_num // Op_RegL
4731 };
4732 // Excluded flags and vector registers.
4733 assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
4734 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4735 %}
4736 %}
4737
4738 //----------ATTRIBUTES---------------------------------------------------------
4739 //----------Operand Attributes-------------------------------------------------
4740 op_attrib op_cost(0); // Required cost attribute
4741
4742 //----------Instruction Attributes---------------------------------------------
4743 ins_attrib ins_cost(100); // Required cost attribute
4744 ins_attrib ins_size(8); // Required size attribute (in bits)
4745 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4746 // a non-matching short branch variant
4747 // of some long branch?
4748 ins_attrib ins_alignment(1); // Required alignment attribute (must
4749 // be a power of 2) specifies the
4750 // alignment that some part of the
4751 // instruction (not necessarily the
4752 // start) requires. If > 1, a
4753 // compute_padding() function must be
4754 // provided for the instruction
4755
4756 // Whether this node is expanded during code emission into a sequence of
4757 // instructions and the first instruction can perform an implicit null check.
4758 ins_attrib ins_is_late_expanded_null_check_candidate(false);
4759
4760 //----------OPERANDS-----------------------------------------------------------
4761 // Operand definitions must precede instruction definitions for correct parsing
4762 // in the ADLC because operands constitute user defined types which are used in
4763 // instruction definitions.
4764
4765 //----------Simple Operands----------------------------------------------------
4766 // Immediate Operands
4767 // Integer Immediate
4768 operand immI()
4769 %{
4770 match(ConI);
4771
4772 op_cost(10);
4773 format %{ %}
4774 interface(CONST_INTER);
4775 %}
4776
4777 // Constant for test vs zero
4778 operand immI_0()
4779 %{
4780 predicate(n->get_int() == 0);
4781 match(ConI);
4782
4783 op_cost(0);
4784 format %{ %}
4785 interface(CONST_INTER);
4786 %}
4787
4788 // Constant for increment
4789 operand immI_1()
4790 %{
4791 predicate(n->get_int() == 1);
4792 match(ConI);
4793
4794 op_cost(0);
4795 format %{ %}
4796 interface(CONST_INTER);
4797 %}
4798
4799 // Constant for decrement
4800 operand immI_M1()
4801 %{
4802 predicate(n->get_int() == -1);
4803 match(ConI);
4804
4805 op_cost(0);
4806 format %{ %}
4807 interface(CONST_INTER);
4808 %}
4809
4810 operand immI_2()
4811 %{
4812 predicate(n->get_int() == 2);
4813 match(ConI);
4814
4815 op_cost(0);
4816 format %{ %}
4817 interface(CONST_INTER);
4818 %}
4819
4820 operand immI_4()
4821 %{
4822 predicate(n->get_int() == 4);
4823 match(ConI);
4824
4825 op_cost(0);
4826 format %{ %}
4827 interface(CONST_INTER);
4828 %}
4829
4830 operand immI_8()
4831 %{
4832 predicate(n->get_int() == 8);
4833 match(ConI);
4834
4835 op_cost(0);
4836 format %{ %}
4837 interface(CONST_INTER);
4838 %}
4839
4840 // Valid scale values for addressing modes
4841 operand immI2()
4842 %{
4843 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4844 match(ConI);
4845
4846 format %{ %}
4847 interface(CONST_INTER);
4848 %}
4849
4850 operand immU7()
4851 %{
4852 predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
4853 match(ConI);
4854
4855 op_cost(5);
4856 format %{ %}
4857 interface(CONST_INTER);
4858 %}
4859
4860 operand immI8()
4861 %{
4862 predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
4863 match(ConI);
4864
4865 op_cost(5);
4866 format %{ %}
4867 interface(CONST_INTER);
4868 %}
4869
4870 operand immU8()
4871 %{
4872 predicate((0 <= n->get_int()) && (n->get_int() <= 255));
4873 match(ConI);
4874
4875 op_cost(5);
4876 format %{ %}
4877 interface(CONST_INTER);
4878 %}
4879
4880 operand immI16()
4881 %{
4882 predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
4883 match(ConI);
4884
4885 op_cost(10);
4886 format %{ %}
4887 interface(CONST_INTER);
4888 %}
4889
4890 // Int Immediate non-negative
4891 operand immU31()
4892 %{
4893 predicate(n->get_int() >= 0);
4894 match(ConI);
4895
4896 op_cost(0);
4897 format %{ %}
4898 interface(CONST_INTER);
4899 %}
4900
4901 // Pointer Immediate
4902 operand immP()
4903 %{
4904 match(ConP);
4905
4906 op_cost(10);
4907 format %{ %}
4908 interface(CONST_INTER);
4909 %}
4910
4911 // Null Pointer Immediate
4912 operand immP0()
4913 %{
4914 predicate(n->get_ptr() == 0);
4915 match(ConP);
4916
4917 op_cost(5);
4918 format %{ %}
4919 interface(CONST_INTER);
4920 %}
4921
4922 // Pointer Immediate
4923 operand immN() %{
4924 match(ConN);
4925
4926 op_cost(10);
4927 format %{ %}
4928 interface(CONST_INTER);
4929 %}
4930
4931 operand immNKlass() %{
4932 match(ConNKlass);
4933
4934 op_cost(10);
4935 format %{ %}
4936 interface(CONST_INTER);
4937 %}
4938
4939 // Null Pointer Immediate
4940 operand immN0() %{
4941 predicate(n->get_narrowcon() == 0);
4942 match(ConN);
4943
4944 op_cost(5);
4945 format %{ %}
4946 interface(CONST_INTER);
4947 %}
4948
4949 operand immP31()
4950 %{
4951 predicate(n->as_Type()->type()->is_ptr()->reloc() == relocInfo::none
4952 && (n->get_ptr() >> 31) == 0);
4953 match(ConP);
4954
4955 op_cost(5);
4956 format %{ %}
4957 interface(CONST_INTER);
4958 %}
4959
4960
4961 // Long Immediate
4962 operand immL()
4963 %{
4964 match(ConL);
4965
4966 op_cost(20);
4967 format %{ %}
4968 interface(CONST_INTER);
4969 %}
4970
4971 // Long Immediate 8-bit
4972 operand immL8()
4973 %{
4974 predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
4975 match(ConL);
4976
4977 op_cost(5);
4978 format %{ %}
4979 interface(CONST_INTER);
4980 %}
4981
4982 // Long Immediate 32-bit unsigned
4983 operand immUL32()
4984 %{
4985 predicate(n->get_long() == (unsigned int) (n->get_long()));
4986 match(ConL);
4987
4988 op_cost(10);
4989 format %{ %}
4990 interface(CONST_INTER);
4991 %}
4992
4993 // Long Immediate 32-bit signed
4994 operand immL32()
4995 %{
4996 predicate(n->get_long() == (int) (n->get_long()));
4997 match(ConL);
4998
4999 op_cost(15);
5000 format %{ %}
5001 interface(CONST_INTER);
5002 %}
5003
5004 operand immL_Pow2()
5005 %{
5006 predicate(is_power_of_2((julong)n->get_long()));
5007 match(ConL);
5008
5009 op_cost(15);
5010 format %{ %}
5011 interface(CONST_INTER);
5012 %}
5013
5014 operand immL_NotPow2()
5015 %{
5016 predicate(is_power_of_2((julong)~n->get_long()));
5017 match(ConL);
5018
5019 op_cost(15);
5020 format %{ %}
5021 interface(CONST_INTER);
5022 %}
5023
5024 // Long Immediate zero
5025 operand immL0()
5026 %{
5027 predicate(n->get_long() == 0L);
5028 match(ConL);
5029
5030 op_cost(10);
5031 format %{ %}
5032 interface(CONST_INTER);
5033 %}
5034
5035 // Constant for increment
5036 operand immL1()
5037 %{
5038 predicate(n->get_long() == 1);
5039 match(ConL);
5040
5041 format %{ %}
5042 interface(CONST_INTER);
5043 %}
5044
5045 // Constant for decrement
5046 operand immL_M1()
5047 %{
5048 predicate(n->get_long() == -1);
5049 match(ConL);
5050
5051 format %{ %}
5052 interface(CONST_INTER);
5053 %}
5054
5055 // Long Immediate: low 32-bit mask
5056 operand immL_32bits()
5057 %{
5058 predicate(n->get_long() == 0xFFFFFFFFL);
5059 match(ConL);
5060 op_cost(20);
5061
5062 format %{ %}
5063 interface(CONST_INTER);
5064 %}
5065
5066 // Int Immediate: 2^n-1, positive
5067 operand immI_Pow2M1()
5068 %{
5069 predicate((n->get_int() > 0)
5070 && is_power_of_2((juint)n->get_int() + 1));
5071 match(ConI);
5072
5073 op_cost(20);
5074 format %{ %}
5075 interface(CONST_INTER);
5076 %}
5077
5078 // Float Immediate zero
5079 operand immF0()
5080 %{
5081 predicate(jint_cast(n->getf()) == 0);
5082 match(ConF);
5083
5084 op_cost(5);
5085 format %{ %}
5086 interface(CONST_INTER);
5087 %}
5088
5089 // Float Immediate
5090 operand immF()
5091 %{
5092 match(ConF);
5093
5094 op_cost(15);
5095 format %{ %}
5096 interface(CONST_INTER);
5097 %}
5098
5099 // Half Float Immediate
5100 operand immH()
5101 %{
5102 match(ConH);
5103
5104 op_cost(15);
5105 format %{ %}
5106 interface(CONST_INTER);
5107 %}
5108
5109 // Double Immediate zero
5110 operand immD0()
5111 %{
5112 predicate(jlong_cast(n->getd()) == 0);
5113 match(ConD);
5114
5115 op_cost(5);
5116 format %{ %}
5117 interface(CONST_INTER);
5118 %}
5119
5120 // Double Immediate
5121 operand immD()
5122 %{
5123 match(ConD);
5124
5125 op_cost(15);
5126 format %{ %}
5127 interface(CONST_INTER);
5128 %}
5129
5130 // Immediates for special shifts (sign extend)
5131
5132 // Constants for increment
5133 operand immI_16()
5134 %{
5135 predicate(n->get_int() == 16);
5136 match(ConI);
5137
5138 format %{ %}
5139 interface(CONST_INTER);
5140 %}
5141
5142 operand immI_24()
5143 %{
5144 predicate(n->get_int() == 24);
5145 match(ConI);
5146
5147 format %{ %}
5148 interface(CONST_INTER);
5149 %}
5150
5151 // Constant for byte-wide masking
5152 operand immI_255()
5153 %{
5154 predicate(n->get_int() == 255);
5155 match(ConI);
5156
5157 format %{ %}
5158 interface(CONST_INTER);
5159 %}
5160
5161 // Constant for short-wide masking
5162 operand immI_65535()
5163 %{
5164 predicate(n->get_int() == 65535);
5165 match(ConI);
5166
5167 format %{ %}
5168 interface(CONST_INTER);
5169 %}
5170
5171 // Constant for byte-wide masking
5172 operand immL_255()
5173 %{
5174 predicate(n->get_long() == 255);
5175 match(ConL);
5176
5177 format %{ %}
5178 interface(CONST_INTER);
5179 %}
5180
5181 // Constant for short-wide masking
5182 operand immL_65535()
5183 %{
5184 predicate(n->get_long() == 65535);
5185 match(ConL);
5186
5187 format %{ %}
5188 interface(CONST_INTER);
5189 %}
5190
5191 // AOT Runtime Constants Address
5192 operand immAOTRuntimeConstantsAddress()
5193 %{
5194 // Check if the address is in the range of AOT Runtime Constants
5195 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
5196 match(ConP);
5197
5198 op_cost(0);
5199 format %{ %}
5200 interface(CONST_INTER);
5201 %}
5202
5203 operand kReg()
5204 %{
5205 constraint(ALLOC_IN_RC(vectmask_reg));
5206 match(RegVectMask);
5207 format %{%}
5208 interface(REG_INTER);
5209 %}
5210
5211 // Register Operands
5212 // Integer Register
5213 operand rRegI()
5214 %{
5215 constraint(ALLOC_IN_RC(int_reg));
5216 match(RegI);
5217
5218 match(rax_RegI);
5219 match(rbx_RegI);
5220 match(rcx_RegI);
5221 match(rdx_RegI);
5222 match(rdi_RegI);
5223
5224 format %{ %}
5225 interface(REG_INTER);
5226 %}
5227
5228 // Special Registers
5229 operand rax_RegI()
5230 %{
5231 constraint(ALLOC_IN_RC(int_rax_reg));
5232 match(RegI);
5233 match(rRegI);
5234
5235 format %{ "RAX" %}
5236 interface(REG_INTER);
5237 %}
5238
5239 // Special Registers
5240 operand rbx_RegI()
5241 %{
5242 constraint(ALLOC_IN_RC(int_rbx_reg));
5243 match(RegI);
5244 match(rRegI);
5245
5246 format %{ "RBX" %}
5247 interface(REG_INTER);
5248 %}
5249
5250 operand rcx_RegI()
5251 %{
5252 constraint(ALLOC_IN_RC(int_rcx_reg));
5253 match(RegI);
5254 match(rRegI);
5255
5256 format %{ "RCX" %}
5257 interface(REG_INTER);
5258 %}
5259
5260 operand rdx_RegI()
5261 %{
5262 constraint(ALLOC_IN_RC(int_rdx_reg));
5263 match(RegI);
5264 match(rRegI);
5265
5266 format %{ "RDX" %}
5267 interface(REG_INTER);
5268 %}
5269
5270 operand rdi_RegI()
5271 %{
5272 constraint(ALLOC_IN_RC(int_rdi_reg));
5273 match(RegI);
5274 match(rRegI);
5275
5276 format %{ "RDI" %}
5277 interface(REG_INTER);
5278 %}
5279
5280 operand no_rax_rdx_RegI()
5281 %{
5282 constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
5283 match(RegI);
5284 match(rbx_RegI);
5285 match(rcx_RegI);
5286 match(rdi_RegI);
5287
5288 format %{ %}
5289 interface(REG_INTER);
5290 %}
5291
5292 operand no_rbp_r13_RegI()
5293 %{
5294 constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
5295 match(RegI);
5296 match(rRegI);
5297 match(rax_RegI);
5298 match(rbx_RegI);
5299 match(rcx_RegI);
5300 match(rdx_RegI);
5301 match(rdi_RegI);
5302
5303 format %{ %}
5304 interface(REG_INTER);
5305 %}
5306
5307 // Pointer Register
5308 operand any_RegP()
5309 %{
5310 constraint(ALLOC_IN_RC(any_reg));
5311 match(RegP);
5312 match(rax_RegP);
5313 match(rbx_RegP);
5314 match(rdi_RegP);
5315 match(rsi_RegP);
5316 match(rbp_RegP);
5317 match(r15_RegP);
5318 match(rRegP);
5319
5320 format %{ %}
5321 interface(REG_INTER);
5322 %}
5323
5324 operand rRegP()
5325 %{
5326 constraint(ALLOC_IN_RC(ptr_reg));
5327 match(RegP);
5328 match(rax_RegP);
5329 match(rbx_RegP);
5330 match(rdi_RegP);
5331 match(rsi_RegP);
5332 match(rbp_RegP); // See Q&A below about
5333 match(r15_RegP); // r15_RegP and rbp_RegP.
5334
5335 format %{ %}
5336 interface(REG_INTER);
5337 %}
5338
5339 operand rRegN() %{
5340 constraint(ALLOC_IN_RC(int_reg));
5341 match(RegN);
5342
5343 format %{ %}
5344 interface(REG_INTER);
5345 %}
5346
5347 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
5348 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
5349 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
5350 // The output of an instruction is controlled by the allocator, which respects
5351 // register class masks, not match rules. Unless an instruction mentions
5352 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
5353 // by the allocator as an input.
5354 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
5355 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
5356 // result, RBP is not included in the output of the instruction either.
5357
5358 // This operand is not allowed to use RBP even if
5359 // RBP is not used to hold the frame pointer.
5360 operand no_rbp_RegP()
5361 %{
5362 constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
5363 match(RegP);
5364 match(rbx_RegP);
5365 match(rsi_RegP);
5366 match(rdi_RegP);
5367
5368 format %{ %}
5369 interface(REG_INTER);
5370 %}
5371
5372 // Special Registers
5373 // Return a pointer value
5374 operand rax_RegP()
5375 %{
5376 constraint(ALLOC_IN_RC(ptr_rax_reg));
5377 match(RegP);
5378 match(rRegP);
5379
5380 format %{ %}
5381 interface(REG_INTER);
5382 %}
5383
5384 // Special Registers
5385 // Return a compressed pointer value
5386 operand rax_RegN()
5387 %{
5388 constraint(ALLOC_IN_RC(int_rax_reg));
5389 match(RegN);
5390 match(rRegN);
5391
5392 format %{ %}
5393 interface(REG_INTER);
5394 %}
5395
5396 // Used in AtomicAdd
5397 operand rbx_RegP()
5398 %{
5399 constraint(ALLOC_IN_RC(ptr_rbx_reg));
5400 match(RegP);
5401 match(rRegP);
5402
5403 format %{ %}
5404 interface(REG_INTER);
5405 %}
5406
5407 operand rsi_RegP()
5408 %{
5409 constraint(ALLOC_IN_RC(ptr_rsi_reg));
5410 match(RegP);
5411 match(rRegP);
5412
5413 format %{ %}
5414 interface(REG_INTER);
5415 %}
5416
5417 operand rbp_RegP()
5418 %{
5419 constraint(ALLOC_IN_RC(ptr_rbp_reg));
5420 match(RegP);
5421 match(rRegP);
5422
5423 format %{ %}
5424 interface(REG_INTER);
5425 %}
5426
5427 // Used in rep stosq
5428 operand rdi_RegP()
5429 %{
5430 constraint(ALLOC_IN_RC(ptr_rdi_reg));
5431 match(RegP);
5432 match(rRegP);
5433
5434 format %{ %}
5435 interface(REG_INTER);
5436 %}
5437
5438 operand r15_RegP()
5439 %{
5440 constraint(ALLOC_IN_RC(ptr_r15_reg));
5441 match(RegP);
5442 match(rRegP);
5443
5444 format %{ %}
5445 interface(REG_INTER);
5446 %}
5447
5448 operand rRegL()
5449 %{
5450 constraint(ALLOC_IN_RC(long_reg));
5451 match(RegL);
5452 match(rax_RegL);
5453 match(rdx_RegL);
5454
5455 format %{ %}
5456 interface(REG_INTER);
5457 %}
5458
5459 // Special Registers
5460 operand no_rax_rdx_RegL()
5461 %{
5462 constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
5463 match(RegL);
5464 match(rRegL);
5465
5466 format %{ %}
5467 interface(REG_INTER);
5468 %}
5469
5470 operand rax_RegL()
5471 %{
5472 constraint(ALLOC_IN_RC(long_rax_reg));
5473 match(RegL);
5474 match(rRegL);
5475
5476 format %{ "RAX" %}
5477 interface(REG_INTER);
5478 %}
5479
5480 operand rcx_RegL()
5481 %{
5482 constraint(ALLOC_IN_RC(long_rcx_reg));
5483 match(RegL);
5484 match(rRegL);
5485
5486 format %{ %}
5487 interface(REG_INTER);
5488 %}
5489
5490 operand rdx_RegL()
5491 %{
5492 constraint(ALLOC_IN_RC(long_rdx_reg));
5493 match(RegL);
5494 match(rRegL);
5495
5496 format %{ %}
5497 interface(REG_INTER);
5498 %}
5499
5500 operand r11_RegL()
5501 %{
5502 constraint(ALLOC_IN_RC(long_r11_reg));
5503 match(RegL);
5504 match(rRegL);
5505
5506 format %{ %}
5507 interface(REG_INTER);
5508 %}
5509
5510 operand no_rbp_r13_RegL()
5511 %{
5512 constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
5513 match(RegL);
5514 match(rRegL);
5515 match(rax_RegL);
5516 match(rcx_RegL);
5517 match(rdx_RegL);
5518
5519 format %{ %}
5520 interface(REG_INTER);
5521 %}
5522
5523 // Flags register, used as output of compare instructions
5524 operand rFlagsReg()
5525 %{
5526 constraint(ALLOC_IN_RC(int_flags));
5527 match(RegFlags);
5528
5529 format %{ "RFLAGS" %}
5530 interface(REG_INTER);
5531 %}
5532
5533 // Flags register, used as output of FLOATING POINT compare instructions
5534 operand rFlagsRegU()
5535 %{
5536 constraint(ALLOC_IN_RC(int_flags));
5537 match(RegFlags);
5538
5539 format %{ "RFLAGS_U" %}
5540 interface(REG_INTER);
5541 %}
5542
5543 operand rFlagsRegUCF() %{
5544 constraint(ALLOC_IN_RC(int_flags));
5545 match(RegFlags);
5546 predicate(!UseAPX || !VM_Version::supports_avx10_2());
5547
5548 format %{ "RFLAGS_U_CF" %}
5549 interface(REG_INTER);
5550 %}
5551
5552 operand rFlagsRegUCFE() %{
5553 constraint(ALLOC_IN_RC(int_flags));
5554 match(RegFlags);
5555 predicate(UseAPX && VM_Version::supports_avx10_2());
5556
5557 format %{ "RFLAGS_U_CFE" %}
5558 interface(REG_INTER);
5559 %}
5560
5561 // Float register operands
5562 operand regF() %{
5563 constraint(ALLOC_IN_RC(float_reg));
5564 match(RegF);
5565
5566 format %{ %}
5567 interface(REG_INTER);
5568 %}
5569
5570 // Float register operands
5571 operand legRegF() %{
5572 constraint(ALLOC_IN_RC(float_reg_legacy));
5573 match(RegF);
5574
5575 format %{ %}
5576 interface(REG_INTER);
5577 %}
5578
5579 // Float register operands
5580 operand vlRegF() %{
5581 constraint(ALLOC_IN_RC(float_reg_vl));
5582 match(RegF);
5583
5584 format %{ %}
5585 interface(REG_INTER);
5586 %}
5587
5588 // Double register operands
5589 operand regD() %{
5590 constraint(ALLOC_IN_RC(double_reg));
5591 match(RegD);
5592
5593 format %{ %}
5594 interface(REG_INTER);
5595 %}
5596
5597 // Double register operands
5598 operand legRegD() %{
5599 constraint(ALLOC_IN_RC(double_reg_legacy));
5600 match(RegD);
5601
5602 format %{ %}
5603 interface(REG_INTER);
5604 %}
5605
5606 // Double register operands
5607 operand vlRegD() %{
5608 constraint(ALLOC_IN_RC(double_reg_vl));
5609 match(RegD);
5610
5611 format %{ %}
5612 interface(REG_INTER);
5613 %}
5614
5615 //----------Memory Operands----------------------------------------------------
5616 // Direct Memory Operand
5617 // operand direct(immP addr)
5618 // %{
5619 // match(addr);
5620
5621 // format %{ "[$addr]" %}
5622 // interface(MEMORY_INTER) %{
5623 // base(0xFFFFFFFF);
5624 // index(0x4);
5625 // scale(0x0);
5626 // disp($addr);
5627 // %}
5628 // %}
5629
5630 // Indirect Memory Operand
5631 operand indirect(any_RegP reg)
5632 %{
5633 constraint(ALLOC_IN_RC(ptr_reg));
5634 match(reg);
5635
5636 format %{ "[$reg]" %}
5637 interface(MEMORY_INTER) %{
5638 base($reg);
5639 index(0x4);
5640 scale(0x0);
5641 disp(0x0);
5642 %}
5643 %}
5644
5645 // Indirect Memory Plus Short Offset Operand
5646 operand indOffset8(any_RegP reg, immL8 off)
5647 %{
5648 constraint(ALLOC_IN_RC(ptr_reg));
5649 match(AddP reg off);
5650
5651 format %{ "[$reg + $off (8-bit)]" %}
5652 interface(MEMORY_INTER) %{
5653 base($reg);
5654 index(0x4);
5655 scale(0x0);
5656 disp($off);
5657 %}
5658 %}
5659
5660 // Indirect Memory Plus Long Offset Operand
5661 operand indOffset32(any_RegP reg, immL32 off)
5662 %{
5663 constraint(ALLOC_IN_RC(ptr_reg));
5664 match(AddP reg off);
5665
5666 format %{ "[$reg + $off (32-bit)]" %}
5667 interface(MEMORY_INTER) %{
5668 base($reg);
5669 index(0x4);
5670 scale(0x0);
5671 disp($off);
5672 %}
5673 %}
5674
5675 // Indirect Memory Plus Index Register Plus Offset Operand
5676 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
5677 %{
5678 constraint(ALLOC_IN_RC(ptr_reg));
5679 match(AddP (AddP reg lreg) off);
5680
5681 op_cost(10);
5682 format %{"[$reg + $off + $lreg]" %}
5683 interface(MEMORY_INTER) %{
5684 base($reg);
5685 index($lreg);
5686 scale(0x0);
5687 disp($off);
5688 %}
5689 %}
5690
5691 // Indirect Memory Plus Index Register Plus Offset Operand
5692 operand indIndex(any_RegP reg, rRegL lreg)
5693 %{
5694 constraint(ALLOC_IN_RC(ptr_reg));
5695 match(AddP reg lreg);
5696
5697 op_cost(10);
5698 format %{"[$reg + $lreg]" %}
5699 interface(MEMORY_INTER) %{
5700 base($reg);
5701 index($lreg);
5702 scale(0x0);
5703 disp(0x0);
5704 %}
5705 %}
5706
5707 // Indirect Memory Times Scale Plus Index Register
5708 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
5709 %{
5710 constraint(ALLOC_IN_RC(ptr_reg));
5711 match(AddP reg (LShiftL lreg scale));
5712
5713 op_cost(10);
5714 format %{"[$reg + $lreg << $scale]" %}
5715 interface(MEMORY_INTER) %{
5716 base($reg);
5717 index($lreg);
5718 scale($scale);
5719 disp(0x0);
5720 %}
5721 %}
5722
5723 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
5724 %{
5725 constraint(ALLOC_IN_RC(ptr_reg));
5726 predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
5727 match(AddP reg (LShiftL (ConvI2L idx) scale));
5728
5729 op_cost(10);
5730 format %{"[$reg + pos $idx << $scale]" %}
5731 interface(MEMORY_INTER) %{
5732 base($reg);
5733 index($idx);
5734 scale($scale);
5735 disp(0x0);
5736 %}
5737 %}
5738
5739 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
5740 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
5741 %{
5742 constraint(ALLOC_IN_RC(ptr_reg));
5743 match(AddP (AddP reg (LShiftL lreg scale)) off);
5744
5745 op_cost(10);
5746 format %{"[$reg + $off + $lreg << $scale]" %}
5747 interface(MEMORY_INTER) %{
5748 base($reg);
5749 index($lreg);
5750 scale($scale);
5751 disp($off);
5752 %}
5753 %}
5754
5755 // Indirect Memory Plus Positive Index Register Plus Offset Operand
5756 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
5757 %{
5758 constraint(ALLOC_IN_RC(ptr_reg));
5759 predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
5760 match(AddP (AddP reg (ConvI2L idx)) off);
5761
5762 op_cost(10);
5763 format %{"[$reg + $off + $idx]" %}
5764 interface(MEMORY_INTER) %{
5765 base($reg);
5766 index($idx);
5767 scale(0x0);
5768 disp($off);
5769 %}
5770 %}
5771
5772 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
5773 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
5774 %{
5775 constraint(ALLOC_IN_RC(ptr_reg));
5776 predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
5777 match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
5778
5779 op_cost(10);
5780 format %{"[$reg + $off + $idx << $scale]" %}
5781 interface(MEMORY_INTER) %{
5782 base($reg);
5783 index($idx);
5784 scale($scale);
5785 disp($off);
5786 %}
5787 %}
5788
5789 // Indirect Narrow Oop Plus Offset Operand
5790 // Note: x86 architecture doesn't support "scale * index + offset" without a base
5791 // we can't free r12 even with CompressedOops::base() == nullptr.
5792 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
5793 predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
5794 constraint(ALLOC_IN_RC(ptr_reg));
5795 match(AddP (DecodeN reg) off);
5796
5797 op_cost(10);
5798 format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
5799 interface(MEMORY_INTER) %{
5800 base(0xc); // R12
5801 index($reg);
5802 scale(0x3);
5803 disp($off);
5804 %}
5805 %}
5806
5807 // Indirect Memory Operand
5808 operand indirectNarrow(rRegN reg)
5809 %{
5810 predicate(CompressedOops::shift() == 0);
5811 constraint(ALLOC_IN_RC(ptr_reg));
5812 match(DecodeN reg);
5813
5814 format %{ "[$reg]" %}
5815 interface(MEMORY_INTER) %{
5816 base($reg);
5817 index(0x4);
5818 scale(0x0);
5819 disp(0x0);
5820 %}
5821 %}
5822
5823 // Indirect Memory Plus Short Offset Operand
5824 operand indOffset8Narrow(rRegN reg, immL8 off)
5825 %{
5826 predicate(CompressedOops::shift() == 0);
5827 constraint(ALLOC_IN_RC(ptr_reg));
5828 match(AddP (DecodeN reg) off);
5829
5830 format %{ "[$reg + $off (8-bit)]" %}
5831 interface(MEMORY_INTER) %{
5832 base($reg);
5833 index(0x4);
5834 scale(0x0);
5835 disp($off);
5836 %}
5837 %}
5838
5839 // Indirect Memory Plus Long Offset Operand
5840 operand indOffset32Narrow(rRegN reg, immL32 off)
5841 %{
5842 predicate(CompressedOops::shift() == 0);
5843 constraint(ALLOC_IN_RC(ptr_reg));
5844 match(AddP (DecodeN reg) off);
5845
5846 format %{ "[$reg + $off (32-bit)]" %}
5847 interface(MEMORY_INTER) %{
5848 base($reg);
5849 index(0x4);
5850 scale(0x0);
5851 disp($off);
5852 %}
5853 %}
5854
5855 // Indirect Memory Plus Index Register Plus Offset Operand
5856 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
5857 %{
5858 predicate(CompressedOops::shift() == 0);
5859 constraint(ALLOC_IN_RC(ptr_reg));
5860 match(AddP (AddP (DecodeN reg) lreg) off);
5861
5862 op_cost(10);
5863 format %{"[$reg + $off + $lreg]" %}
5864 interface(MEMORY_INTER) %{
5865 base($reg);
5866 index($lreg);
5867 scale(0x0);
5868 disp($off);
5869 %}
5870 %}
5871
5872 // Indirect Memory Plus Index Register Plus Offset Operand
5873 operand indIndexNarrow(rRegN reg, rRegL lreg)
5874 %{
5875 predicate(CompressedOops::shift() == 0);
5876 constraint(ALLOC_IN_RC(ptr_reg));
5877 match(AddP (DecodeN reg) lreg);
5878
5879 op_cost(10);
5880 format %{"[$reg + $lreg]" %}
5881 interface(MEMORY_INTER) %{
5882 base($reg);
5883 index($lreg);
5884 scale(0x0);
5885 disp(0x0);
5886 %}
5887 %}
5888
5889 // Indirect Memory Times Scale Plus Index Register
5890 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
5891 %{
5892 predicate(CompressedOops::shift() == 0);
5893 constraint(ALLOC_IN_RC(ptr_reg));
5894 match(AddP (DecodeN reg) (LShiftL lreg scale));
5895
5896 op_cost(10);
5897 format %{"[$reg + $lreg << $scale]" %}
5898 interface(MEMORY_INTER) %{
5899 base($reg);
5900 index($lreg);
5901 scale($scale);
5902 disp(0x0);
5903 %}
5904 %}
5905
5906 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
5907 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
5908 %{
5909 predicate(CompressedOops::shift() == 0);
5910 constraint(ALLOC_IN_RC(ptr_reg));
5911 match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
5912
5913 op_cost(10);
5914 format %{"[$reg + $off + $lreg << $scale]" %}
5915 interface(MEMORY_INTER) %{
5916 base($reg);
5917 index($lreg);
5918 scale($scale);
5919 disp($off);
5920 %}
5921 %}
5922
5923 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
5924 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
5925 %{
5926 constraint(ALLOC_IN_RC(ptr_reg));
5927 predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
5928 match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
5929
5930 op_cost(10);
5931 format %{"[$reg + $off + $idx]" %}
5932 interface(MEMORY_INTER) %{
5933 base($reg);
5934 index($idx);
5935 scale(0x0);
5936 disp($off);
5937 %}
5938 %}
5939
5940 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
5941 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
5942 %{
5943 constraint(ALLOC_IN_RC(ptr_reg));
5944 predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
5945 match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
5946
5947 op_cost(10);
5948 format %{"[$reg + $off + $idx << $scale]" %}
5949 interface(MEMORY_INTER) %{
5950 base($reg);
5951 index($idx);
5952 scale($scale);
5953 disp($off);
5954 %}
5955 %}
5956
5957 //----------Special Memory Operands--------------------------------------------
5958 // Stack Slot Operand - This operand is used for loading and storing temporary
5959 // values on the stack where a match requires a value to
5960 // flow through memory.
5961 operand stackSlotP(sRegP reg)
5962 %{
5963 constraint(ALLOC_IN_RC(stack_slots));
5964 // No match rule because this operand is only generated in matching
5965
5966 format %{ "[$reg]" %}
5967 interface(MEMORY_INTER) %{
5968 base(0x4); // RSP
5969 index(0x4); // No Index
5970 scale(0x0); // No Scale
5971 disp($reg); // Stack Offset
5972 %}
5973 %}
5974
5975 operand stackSlotI(sRegI reg)
5976 %{
5977 constraint(ALLOC_IN_RC(stack_slots));
5978 // No match rule because this operand is only generated in matching
5979
5980 format %{ "[$reg]" %}
5981 interface(MEMORY_INTER) %{
5982 base(0x4); // RSP
5983 index(0x4); // No Index
5984 scale(0x0); // No Scale
5985 disp($reg); // Stack Offset
5986 %}
5987 %}
5988
5989 operand stackSlotF(sRegF reg)
5990 %{
5991 constraint(ALLOC_IN_RC(stack_slots));
5992 // No match rule because this operand is only generated in matching
5993
5994 format %{ "[$reg]" %}
5995 interface(MEMORY_INTER) %{
5996 base(0x4); // RSP
5997 index(0x4); // No Index
5998 scale(0x0); // No Scale
5999 disp($reg); // Stack Offset
6000 %}
6001 %}
6002
6003 operand stackSlotD(sRegD reg)
6004 %{
6005 constraint(ALLOC_IN_RC(stack_slots));
6006 // No match rule because this operand is only generated in matching
6007
6008 format %{ "[$reg]" %}
6009 interface(MEMORY_INTER) %{
6010 base(0x4); // RSP
6011 index(0x4); // No Index
6012 scale(0x0); // No Scale
6013 disp($reg); // Stack Offset
6014 %}
6015 %}
6016 operand stackSlotL(sRegL reg)
6017 %{
6018 constraint(ALLOC_IN_RC(stack_slots));
6019 // No match rule because this operand is only generated in matching
6020
6021 format %{ "[$reg]" %}
6022 interface(MEMORY_INTER) %{
6023 base(0x4); // RSP
6024 index(0x4); // No Index
6025 scale(0x0); // No Scale
6026 disp($reg); // Stack Offset
6027 %}
6028 %}
6029
6030 //----------Conditional Branch Operands----------------------------------------
6031 // Comparison Op - This is the operation of the comparison, and is limited to
6032 // the following set of codes:
6033 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6034 //
6035 // Other attributes of the comparison, such as unsignedness, are specified
6036 // by the comparison instruction that sets a condition code flags register.
6037 // That result is represented by a flags operand whose subtype is appropriate
6038 // to the unsignedness (etc.) of the comparison.
6039 //
6040 // Later, the instruction which matches both the Comparison Op (a Bool) and
6041 // the flags (produced by the Cmp) specifies the coding of the comparison op
6042 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6043
6044 // Comparison Code
6045 operand cmpOp()
6046 %{
6047 match(Bool);
6048
6049 format %{ "" %}
6050 interface(COND_INTER) %{
6051 equal(0x4, "e");
6052 not_equal(0x5, "ne");
6053 less(0xc, "l");
6054 greater_equal(0xd, "ge");
6055 less_equal(0xe, "le");
6056 greater(0xf, "g");
6057 overflow(0x0, "o");
6058 no_overflow(0x1, "no");
6059 %}
6060 %}
6061
6062 // Comparison Code, unsigned compare. Used by FP also, with
6063 // C2 (unordered) turned into GT or LT already. The other bits
6064 // C0 and C3 are turned into Carry & Zero flags.
6065 operand cmpOpU()
6066 %{
6067 match(Bool);
6068
6069 format %{ "" %}
6070 interface(COND_INTER) %{
6071 equal(0x4, "e");
6072 not_equal(0x5, "ne");
6073 less(0x2, "b");
6074 greater_equal(0x3, "ae");
6075 less_equal(0x6, "be");
6076 greater(0x7, "a");
6077 overflow(0x0, "o");
6078 no_overflow(0x1, "no");
6079 %}
6080 %}
6081
6082
6083 // Floating comparisons that don't require any fixup for the unordered case,
6084 // If both inputs of the comparison are the same, ZF is always set so we
6085 // don't need to use cmpOpUCF2 for eq/ne
6086 operand cmpOpUCF() %{
6087 match(Bool);
6088 predicate((!UseAPX || !VM_Version::supports_avx10_2()) &&
6089 (n->as_Bool()->_test._test == BoolTest::lt ||
6090 n->as_Bool()->_test._test == BoolTest::ge ||
6091 n->as_Bool()->_test._test == BoolTest::le ||
6092 n->as_Bool()->_test._test == BoolTest::gt ||
6093 n->in(1)->in(1) == n->in(1)->in(2)));
6094 format %{ "" %}
6095 interface(COND_INTER) %{
6096 equal(0xb, "np");
6097 not_equal(0xa, "p");
6098 less(0x2, "b");
6099 greater_equal(0x3, "ae");
6100 less_equal(0x6, "be");
6101 greater(0x7, "a");
6102 overflow(0x0, "o");
6103 no_overflow(0x1, "no");
6104 %}
6105 %}
6106
6107
6108 // Floating comparisons that can be fixed up with extra conditional jumps
6109 operand cmpOpUCF2() %{
6110 match(Bool);
6111 predicate((!UseAPX || !VM_Version::supports_avx10_2()) &&
6112 (n->as_Bool()->_test._test == BoolTest::ne ||
6113 n->as_Bool()->_test._test == BoolTest::eq) &&
6114 n->in(1)->in(1) != n->in(1)->in(2));
6115 format %{ "" %}
6116 interface(COND_INTER) %{
6117 equal(0x4, "e");
6118 not_equal(0x5, "ne");
6119 less(0x2, "b");
6120 greater_equal(0x3, "ae");
6121 less_equal(0x6, "be");
6122 greater(0x7, "a");
6123 overflow(0x0, "o");
6124 no_overflow(0x1, "no");
6125 %}
6126 %}
6127
6128
6129 // Floating point comparisons that set condition flags to test more directly,
6130 // Unsigned tests are used for G (>) and GE (>=) conditions while signed tests
6131 // are used for L (<) and LE (<=) conditions. It's important to convert these
6132 // latter conditions to ones that use unsigned tests before passing into an
6133 // instruction because the preceding comparison might be based on a three way
6134 // comparison (CmpF3 or CmpD3) that also assigns unordered outcomes to -1.
6135 operand cmpOpUCFE()
6136 %{
6137 match(Bool);
6138 predicate((UseAPX && VM_Version::supports_avx10_2()) &&
6139 (n->as_Bool()->_test._test == BoolTest::ne ||
6140 n->as_Bool()->_test._test == BoolTest::eq ||
6141 n->as_Bool()->_test._test == BoolTest::lt ||
6142 n->as_Bool()->_test._test == BoolTest::ge ||
6143 n->as_Bool()->_test._test == BoolTest::le ||
6144 n->as_Bool()->_test._test == BoolTest::gt));
6145
6146 format %{ "" %}
6147 interface(COND_INTER) %{
6148 equal(0x4, "e");
6149 not_equal(0x5, "ne");
6150 less(0x2, "b");
6151 greater_equal(0x3, "ae");
6152 less_equal(0x6, "be");
6153 greater(0x7, "a");
6154 overflow(0x0, "o");
6155 no_overflow(0x1, "no");
6156 %}
6157 %}
6158
6159 // Operands for bound floating pointer register arguments
6160 operand rxmm0() %{
6161 constraint(ALLOC_IN_RC(xmm0_reg));
6162 match(VecX);
6163 format%{%}
6164 interface(REG_INTER);
6165 %}
6166
6167 // Vectors
6168
6169 // Dummy generic vector class. Should be used for all vector operands.
6170 // Replaced with vec[SDXYZ] during post-selection pass.
6171 operand vec() %{
6172 constraint(ALLOC_IN_RC(dynamic));
6173 match(VecX);
6174 match(VecY);
6175 match(VecZ);
6176 match(VecS);
6177 match(VecD);
6178
6179 format %{ %}
6180 interface(REG_INTER);
6181 %}
6182
6183 // Dummy generic legacy vector class. Should be used for all legacy vector operands.
6184 // Replaced with legVec[SDXYZ] during post-selection cleanup.
6185 // Note: legacy register class is used to avoid extra (unneeded in 32-bit VM)
6186 // runtime code generation via reg_class_dynamic.
6187 operand legVec() %{
6188 constraint(ALLOC_IN_RC(dynamic));
6189 match(VecX);
6190 match(VecY);
6191 match(VecZ);
6192 match(VecS);
6193 match(VecD);
6194
6195 format %{ %}
6196 interface(REG_INTER);
6197 %}
6198
6199 // Replaces vec during post-selection cleanup. See above.
6200 operand vecS() %{
6201 constraint(ALLOC_IN_RC(vectors_reg_vlbwdq));
6202 match(VecS);
6203
6204 format %{ %}
6205 interface(REG_INTER);
6206 %}
6207
6208 // Replaces legVec during post-selection cleanup. See above.
6209 operand legVecS() %{
6210 constraint(ALLOC_IN_RC(vectors_reg_legacy));
6211 match(VecS);
6212
6213 format %{ %}
6214 interface(REG_INTER);
6215 %}
6216
6217 // Replaces vec during post-selection cleanup. See above.
6218 operand vecD() %{
6219 constraint(ALLOC_IN_RC(vectord_reg_vlbwdq));
6220 match(VecD);
6221
6222 format %{ %}
6223 interface(REG_INTER);
6224 %}
6225
6226 // Replaces legVec during post-selection cleanup. See above.
6227 operand legVecD() %{
6228 constraint(ALLOC_IN_RC(vectord_reg_legacy));
6229 match(VecD);
6230
6231 format %{ %}
6232 interface(REG_INTER);
6233 %}
6234
6235 // Replaces vec during post-selection cleanup. See above.
6236 operand vecX() %{
6237 constraint(ALLOC_IN_RC(vectorx_reg_vlbwdq));
6238 match(VecX);
6239
6240 format %{ %}
6241 interface(REG_INTER);
6242 %}
6243
6244 // Replaces legVec during post-selection cleanup. See above.
6245 operand legVecX() %{
6246 constraint(ALLOC_IN_RC(vectorx_reg_legacy));
6247 match(VecX);
6248
6249 format %{ %}
6250 interface(REG_INTER);
6251 %}
6252
6253 // Replaces vec during post-selection cleanup. See above.
6254 operand vecY() %{
6255 constraint(ALLOC_IN_RC(vectory_reg_vlbwdq));
6256 match(VecY);
6257
6258 format %{ %}
6259 interface(REG_INTER);
6260 %}
6261
6262 // Replaces legVec during post-selection cleanup. See above.
6263 operand legVecY() %{
6264 constraint(ALLOC_IN_RC(vectory_reg_legacy));
6265 match(VecY);
6266
6267 format %{ %}
6268 interface(REG_INTER);
6269 %}
6270
6271 // Replaces vec during post-selection cleanup. See above.
6272 operand vecZ() %{
6273 constraint(ALLOC_IN_RC(vectorz_reg));
6274 match(VecZ);
6275
6276 format %{ %}
6277 interface(REG_INTER);
6278 %}
6279
6280 // Replaces legVec during post-selection cleanup. See above.
6281 operand legVecZ() %{
6282 constraint(ALLOC_IN_RC(vectorz_reg_legacy));
6283 match(VecZ);
6284
6285 format %{ %}
6286 interface(REG_INTER);
6287 %}
6288
6289 //----------OPERAND CLASSES----------------------------------------------------
6290 // Operand Classes are groups of operands that are used as to simplify
6291 // instruction definitions by not requiring the AD writer to specify separate
6292 // instructions for every form of operand when the instruction accepts
6293 // multiple operand types with the same basic encoding and format. The classic
6294 // case of this is memory operands.
6295
6296 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
6297 indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
6298 indCompressedOopOffset,
6299 indirectNarrow, indOffset8Narrow, indOffset32Narrow,
6300 indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
6301 indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
6302
6303 //----------PIPELINE-----------------------------------------------------------
6304 // Rules which define the behavior of the target architectures pipeline.
6305 pipeline %{
6306
6307 //----------ATTRIBUTES---------------------------------------------------------
6308 attributes %{
6309 variable_size_instructions; // Fixed size instructions
6310 max_instructions_per_bundle = 3; // Up to 3 instructions per bundle
6311 instruction_unit_size = 1; // An instruction is 1 bytes long
6312 instruction_fetch_unit_size = 16; // The processor fetches one line
6313 instruction_fetch_units = 1; // of 16 bytes
6314 %}
6315
6316 //----------RESOURCES----------------------------------------------------------
6317 // Resources are the functional units available to the machine
6318
6319 // Generic P2/P3 pipeline
6320 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
6321 // 3 instructions decoded per cycle.
6322 // 2 load/store ops per cycle, 1 branch, 1 FPU,
6323 // 3 ALU op, only ALU0 handles mul instructions.
6324 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
6325 MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
6326 BR, FPU,
6327 ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
6328
6329 //----------PIPELINE DESCRIPTION-----------------------------------------------
6330 // Pipeline Description specifies the stages in the machine's pipeline
6331
6332 // Generic P2/P3 pipeline
6333 pipe_desc(S0, S1, S2, S3, S4, S5);
6334
6335 //----------PIPELINE CLASSES---------------------------------------------------
6336 // Pipeline Classes describe the stages in which input and output are
6337 // referenced by the hardware pipeline.
6338
6339 // Naming convention: ialu or fpu
6340 // Then: _reg
6341 // Then: _reg if there is a 2nd register
6342 // Then: _long if it's a pair of instructions implementing a long
6343 // Then: _fat if it requires the big decoder
6344 // Or: _mem if it requires the big decoder and a memory unit.
6345
6346 // Integer ALU reg operation
6347 pipe_class ialu_reg(rRegI dst)
6348 %{
6349 single_instruction;
6350 dst : S4(write);
6351 dst : S3(read);
6352 DECODE : S0; // any decoder
6353 ALU : S3; // any alu
6354 %}
6355
6356 // Long ALU reg operation
6357 pipe_class ialu_reg_long(rRegL dst)
6358 %{
6359 instruction_count(2);
6360 dst : S4(write);
6361 dst : S3(read);
6362 DECODE : S0(2); // any 2 decoders
6363 ALU : S3(2); // both alus
6364 %}
6365
6366 // Integer ALU reg operation using big decoder
6367 pipe_class ialu_reg_fat(rRegI dst)
6368 %{
6369 single_instruction;
6370 dst : S4(write);
6371 dst : S3(read);
6372 D0 : S0; // big decoder only
6373 ALU : S3; // any alu
6374 %}
6375
6376 // Integer ALU reg-reg operation
6377 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
6378 %{
6379 single_instruction;
6380 dst : S4(write);
6381 src : S3(read);
6382 DECODE : S0; // any decoder
6383 ALU : S3; // any alu
6384 %}
6385
6386 // Integer ALU reg-reg operation
6387 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
6388 %{
6389 single_instruction;
6390 dst : S4(write);
6391 src : S3(read);
6392 D0 : S0; // big decoder only
6393 ALU : S3; // any alu
6394 %}
6395
6396 // Integer ALU reg-mem operation
6397 pipe_class ialu_reg_mem(rRegI dst, memory mem)
6398 %{
6399 single_instruction;
6400 dst : S5(write);
6401 mem : S3(read);
6402 D0 : S0; // big decoder only
6403 ALU : S4; // any alu
6404 MEM : S3; // any mem
6405 %}
6406
6407 // Integer mem operation (prefetch)
6408 pipe_class ialu_mem(memory mem)
6409 %{
6410 single_instruction;
6411 mem : S3(read);
6412 D0 : S0; // big decoder only
6413 MEM : S3; // any mem
6414 %}
6415
6416 // Integer Store to Memory
6417 pipe_class ialu_mem_reg(memory mem, rRegI src)
6418 %{
6419 single_instruction;
6420 mem : S3(read);
6421 src : S5(read);
6422 D0 : S0; // big decoder only
6423 ALU : S4; // any alu
6424 MEM : S3;
6425 %}
6426
6427 // // Long Store to Memory
6428 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
6429 // %{
6430 // instruction_count(2);
6431 // mem : S3(read);
6432 // src : S5(read);
6433 // D0 : S0(2); // big decoder only; twice
6434 // ALU : S4(2); // any 2 alus
6435 // MEM : S3(2); // Both mems
6436 // %}
6437
6438 // Integer Store to Memory
6439 pipe_class ialu_mem_imm(memory mem)
6440 %{
6441 single_instruction;
6442 mem : S3(read);
6443 D0 : S0; // big decoder only
6444 ALU : S4; // any alu
6445 MEM : S3;
6446 %}
6447
6448 // Integer ALU0 reg-reg operation
6449 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
6450 %{
6451 single_instruction;
6452 dst : S4(write);
6453 src : S3(read);
6454 D0 : S0; // Big decoder only
6455 ALU0 : S3; // only alu0
6456 %}
6457
6458 // Integer ALU0 reg-mem operation
6459 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
6460 %{
6461 single_instruction;
6462 dst : S5(write);
6463 mem : S3(read);
6464 D0 : S0; // big decoder only
6465 ALU0 : S4; // ALU0 only
6466 MEM : S3; // any mem
6467 %}
6468
6469 // Integer ALU reg-reg operation
6470 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
6471 %{
6472 single_instruction;
6473 cr : S4(write);
6474 src1 : S3(read);
6475 src2 : S3(read);
6476 DECODE : S0; // any decoder
6477 ALU : S3; // any alu
6478 %}
6479
6480 // Integer ALU reg-imm operation
6481 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
6482 %{
6483 single_instruction;
6484 cr : S4(write);
6485 src1 : S3(read);
6486 DECODE : S0; // any decoder
6487 ALU : S3; // any alu
6488 %}
6489
6490 // Integer ALU reg-mem operation
6491 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
6492 %{
6493 single_instruction;
6494 cr : S4(write);
6495 src1 : S3(read);
6496 src2 : S3(read);
6497 D0 : S0; // big decoder only
6498 ALU : S4; // any alu
6499 MEM : S3;
6500 %}
6501
6502 // Conditional move reg-reg
6503 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
6504 %{
6505 instruction_count(4);
6506 y : S4(read);
6507 q : S3(read);
6508 p : S3(read);
6509 DECODE : S0(4); // any decoder
6510 %}
6511
6512 // Conditional move reg-reg
6513 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
6514 %{
6515 single_instruction;
6516 dst : S4(write);
6517 src : S3(read);
6518 cr : S3(read);
6519 DECODE : S0; // any decoder
6520 %}
6521
6522 // Conditional move reg-mem
6523 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
6524 %{
6525 single_instruction;
6526 dst : S4(write);
6527 src : S3(read);
6528 cr : S3(read);
6529 DECODE : S0; // any decoder
6530 MEM : S3;
6531 %}
6532
6533 // Conditional move reg-reg long
6534 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
6535 %{
6536 single_instruction;
6537 dst : S4(write);
6538 src : S3(read);
6539 cr : S3(read);
6540 DECODE : S0(2); // any 2 decoders
6541 %}
6542
6543 // Float reg-reg operation
6544 pipe_class fpu_reg(regD dst)
6545 %{
6546 instruction_count(2);
6547 dst : S3(read);
6548 DECODE : S0(2); // any 2 decoders
6549 FPU : S3;
6550 %}
6551
6552 // Float reg-reg operation
6553 pipe_class fpu_reg_reg(regD dst, regD src)
6554 %{
6555 instruction_count(2);
6556 dst : S4(write);
6557 src : S3(read);
6558 DECODE : S0(2); // any 2 decoders
6559 FPU : S3;
6560 %}
6561
6562 // Float reg-reg operation
6563 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
6564 %{
6565 instruction_count(3);
6566 dst : S4(write);
6567 src1 : S3(read);
6568 src2 : S3(read);
6569 DECODE : S0(3); // any 3 decoders
6570 FPU : S3(2);
6571 %}
6572
6573 // Float reg-reg operation
6574 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
6575 %{
6576 instruction_count(4);
6577 dst : S4(write);
6578 src1 : S3(read);
6579 src2 : S3(read);
6580 src3 : S3(read);
6581 DECODE : S0(4); // any 3 decoders
6582 FPU : S3(2);
6583 %}
6584
6585 // Float reg-reg operation
6586 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
6587 %{
6588 instruction_count(4);
6589 dst : S4(write);
6590 src1 : S3(read);
6591 src2 : S3(read);
6592 src3 : S3(read);
6593 DECODE : S1(3); // any 3 decoders
6594 D0 : S0; // Big decoder only
6595 FPU : S3(2);
6596 MEM : S3;
6597 %}
6598
6599 // Float reg-mem operation
6600 pipe_class fpu_reg_mem(regD dst, memory mem)
6601 %{
6602 instruction_count(2);
6603 dst : S5(write);
6604 mem : S3(read);
6605 D0 : S0; // big decoder only
6606 DECODE : S1; // any decoder for FPU POP
6607 FPU : S4;
6608 MEM : S3; // any mem
6609 %}
6610
6611 // Float reg-mem operation
6612 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
6613 %{
6614 instruction_count(3);
6615 dst : S5(write);
6616 src1 : S3(read);
6617 mem : S3(read);
6618 D0 : S0; // big decoder only
6619 DECODE : S1(2); // any decoder for FPU POP
6620 FPU : S4;
6621 MEM : S3; // any mem
6622 %}
6623
6624 // Float mem-reg operation
6625 pipe_class fpu_mem_reg(memory mem, regD src)
6626 %{
6627 instruction_count(2);
6628 src : S5(read);
6629 mem : S3(read);
6630 DECODE : S0; // any decoder for FPU PUSH
6631 D0 : S1; // big decoder only
6632 FPU : S4;
6633 MEM : S3; // any mem
6634 %}
6635
6636 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
6637 %{
6638 instruction_count(3);
6639 src1 : S3(read);
6640 src2 : S3(read);
6641 mem : S3(read);
6642 DECODE : S0(2); // any decoder for FPU PUSH
6643 D0 : S1; // big decoder only
6644 FPU : S4;
6645 MEM : S3; // any mem
6646 %}
6647
6648 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
6649 %{
6650 instruction_count(3);
6651 src1 : S3(read);
6652 src2 : S3(read);
6653 mem : S4(read);
6654 DECODE : S0; // any decoder for FPU PUSH
6655 D0 : S0(2); // big decoder only
6656 FPU : S4;
6657 MEM : S3(2); // any mem
6658 %}
6659
6660 pipe_class fpu_mem_mem(memory dst, memory src1)
6661 %{
6662 instruction_count(2);
6663 src1 : S3(read);
6664 dst : S4(read);
6665 D0 : S0(2); // big decoder only
6666 MEM : S3(2); // any mem
6667 %}
6668
6669 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
6670 %{
6671 instruction_count(3);
6672 src1 : S3(read);
6673 src2 : S3(read);
6674 dst : S4(read);
6675 D0 : S0(3); // big decoder only
6676 FPU : S4;
6677 MEM : S3(3); // any mem
6678 %}
6679
6680 pipe_class fpu_mem_reg_con(memory mem, regD src1)
6681 %{
6682 instruction_count(3);
6683 src1 : S4(read);
6684 mem : S4(read);
6685 DECODE : S0; // any decoder for FPU PUSH
6686 D0 : S0(2); // big decoder only
6687 FPU : S4;
6688 MEM : S3(2); // any mem
6689 %}
6690
6691 // Float load constant
6692 pipe_class fpu_reg_con(regD dst)
6693 %{
6694 instruction_count(2);
6695 dst : S5(write);
6696 D0 : S0; // big decoder only for the load
6697 DECODE : S1; // any decoder for FPU POP
6698 FPU : S4;
6699 MEM : S3; // any mem
6700 %}
6701
6702 // Float load constant
6703 pipe_class fpu_reg_reg_con(regD dst, regD src)
6704 %{
6705 instruction_count(3);
6706 dst : S5(write);
6707 src : S3(read);
6708 D0 : S0; // big decoder only for the load
6709 DECODE : S1(2); // any decoder for FPU POP
6710 FPU : S4;
6711 MEM : S3; // any mem
6712 %}
6713
6714 // UnConditional branch
6715 pipe_class pipe_jmp(label labl)
6716 %{
6717 single_instruction;
6718 BR : S3;
6719 %}
6720
6721 // Conditional branch
6722 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
6723 %{
6724 single_instruction;
6725 cr : S1(read);
6726 BR : S3;
6727 %}
6728
6729 // Allocation idiom
6730 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
6731 %{
6732 instruction_count(1); force_serialization;
6733 fixed_latency(6);
6734 heap_ptr : S3(read);
6735 DECODE : S0(3);
6736 D0 : S2;
6737 MEM : S3;
6738 ALU : S3(2);
6739 dst : S5(write);
6740 BR : S5;
6741 %}
6742
6743 // Generic big/slow expanded idiom
6744 pipe_class pipe_slow()
6745 %{
6746 instruction_count(10); multiple_bundles; force_serialization;
6747 fixed_latency(100);
6748 D0 : S0(2);
6749 MEM : S3(2);
6750 %}
6751
6752 // The real do-nothing guy
6753 pipe_class empty()
6754 %{
6755 instruction_count(0);
6756 %}
6757
6758 // Define the class for the Nop node
6759 define
6760 %{
6761 MachNop = empty;
6762 %}
6763
6764 %}
6765
6766 //----------INSTRUCTIONS-------------------------------------------------------
6767 //
6768 // match -- States which machine-independent subtree may be replaced
6769 // by this instruction.
6770 // ins_cost -- The estimated cost of this instruction is used by instruction
6771 // selection to identify a minimum cost tree of machine
6772 // instructions that matches a tree of machine-independent
6773 // instructions.
6774 // format -- A string providing the disassembly for this instruction.
6775 // The value of an instruction's operand may be inserted
6776 // by referring to it with a '$' prefix.
6777 // opcode -- Three instruction opcodes may be provided. These are referred
6778 // to within an encode class as $primary, $secondary, and $tertiary
6779 // rrspectively. The primary opcode is commonly used to
6780 // indicate the type of machine instruction, while secondary
6781 // and tertiary are often used for prefix options or addressing
6782 // modes.
6783 // ins_encode -- A list of encode classes with parameters. The encode class
6784 // name must have been defined in an 'enc_class' specification
6785 // in the encode section of the architecture description.
6786
6787 // ============================================================================
6788
6789 instruct ShouldNotReachHere() %{
6790 match(Halt);
6791 format %{ "stop\t# ShouldNotReachHere" %}
6792 ins_encode %{
6793 if (is_reachable()) {
6794 const char* str = __ code_string(_halt_reason);
6795 __ stop(str);
6796 }
6797 %}
6798 ins_pipe(pipe_slow);
6799 %}
6800
6801 // ============================================================================
6802
6803 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
6804 // Load Float
6805 instruct MoveF2VL(vlRegF dst, regF src) %{
6806 match(Set dst src);
6807 format %{ "movss $dst,$src\t! load float (4 bytes)" %}
6808 ins_encode %{
6809 ShouldNotReachHere();
6810 %}
6811 ins_pipe( fpu_reg_reg );
6812 %}
6813
6814 // Load Float
6815 instruct MoveF2LEG(legRegF dst, regF src) %{
6816 match(Set dst src);
6817 format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
6818 ins_encode %{
6819 ShouldNotReachHere();
6820 %}
6821 ins_pipe( fpu_reg_reg );
6822 %}
6823
6824 // Load Float
6825 instruct MoveVL2F(regF dst, vlRegF src) %{
6826 match(Set dst src);
6827 format %{ "movss $dst,$src\t! load float (4 bytes)" %}
6828 ins_encode %{
6829 ShouldNotReachHere();
6830 %}
6831 ins_pipe( fpu_reg_reg );
6832 %}
6833
6834 // Load Float
6835 instruct MoveLEG2F(regF dst, legRegF src) %{
6836 match(Set dst src);
6837 format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
6838 ins_encode %{
6839 ShouldNotReachHere();
6840 %}
6841 ins_pipe( fpu_reg_reg );
6842 %}
6843
6844 // Load Double
6845 instruct MoveD2VL(vlRegD dst, regD src) %{
6846 match(Set dst src);
6847 format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
6848 ins_encode %{
6849 ShouldNotReachHere();
6850 %}
6851 ins_pipe( fpu_reg_reg );
6852 %}
6853
6854 // Load Double
6855 instruct MoveD2LEG(legRegD dst, regD src) %{
6856 match(Set dst src);
6857 format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
6858 ins_encode %{
6859 ShouldNotReachHere();
6860 %}
6861 ins_pipe( fpu_reg_reg );
6862 %}
6863
6864 // Load Double
6865 instruct MoveVL2D(regD dst, vlRegD src) %{
6866 match(Set dst src);
6867 format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
6868 ins_encode %{
6869 ShouldNotReachHere();
6870 %}
6871 ins_pipe( fpu_reg_reg );
6872 %}
6873
6874 // Load Double
6875 instruct MoveLEG2D(regD dst, legRegD src) %{
6876 match(Set dst src);
6877 format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
6878 ins_encode %{
6879 ShouldNotReachHere();
6880 %}
6881 ins_pipe( fpu_reg_reg );
6882 %}
6883
6884 //----------Load/Store/Move Instructions---------------------------------------
6885 //----------Load Instructions--------------------------------------------------
6886
6887 // Load Byte (8 bit signed)
6888 instruct loadB(rRegI dst, memory mem)
6889 %{
6890 match(Set dst (LoadB mem));
6891
6892 ins_cost(125);
6893 format %{ "movsbl $dst, $mem\t# byte" %}
6894
6895 ins_encode %{
6896 __ movsbl($dst$$Register, $mem$$Address);
6897 %}
6898
6899 ins_pipe(ialu_reg_mem);
6900 %}
6901
6902 // Load Byte (8 bit signed) into Long Register
6903 instruct loadB2L(rRegL dst, memory mem)
6904 %{
6905 match(Set dst (ConvI2L (LoadB mem)));
6906
6907 ins_cost(125);
6908 format %{ "movsbq $dst, $mem\t# byte -> long" %}
6909
6910 ins_encode %{
6911 __ movsbq($dst$$Register, $mem$$Address);
6912 %}
6913
6914 ins_pipe(ialu_reg_mem);
6915 %}
6916
6917 // Load Unsigned Byte (8 bit UNsigned)
6918 instruct loadUB(rRegI dst, memory mem)
6919 %{
6920 match(Set dst (LoadUB mem));
6921
6922 ins_cost(125);
6923 format %{ "movzbl $dst, $mem\t# ubyte" %}
6924
6925 ins_encode %{
6926 __ movzbl($dst$$Register, $mem$$Address);
6927 %}
6928
6929 ins_pipe(ialu_reg_mem);
6930 %}
6931
6932 // Load Unsigned Byte (8 bit UNsigned) into Long Register
6933 instruct loadUB2L(rRegL dst, memory mem)
6934 %{
6935 match(Set dst (ConvI2L (LoadUB mem)));
6936
6937 ins_cost(125);
6938 format %{ "movzbq $dst, $mem\t# ubyte -> long" %}
6939
6940 ins_encode %{
6941 __ movzbq($dst$$Register, $mem$$Address);
6942 %}
6943
6944 ins_pipe(ialu_reg_mem);
6945 %}
6946
6947 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
6948 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
6949 match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
6950 effect(KILL cr);
6951
6952 format %{ "movzbq $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
6953 "andl $dst, right_n_bits($mask, 8)" %}
6954 ins_encode %{
6955 Register Rdst = $dst$$Register;
6956 __ movzbq(Rdst, $mem$$Address);
6957 __ andl(Rdst, $mask$$constant & right_n_bits(8));
6958 %}
6959 ins_pipe(ialu_reg_mem);
6960 %}
6961
6962 // Load Short (16 bit signed)
6963 instruct loadS(rRegI dst, memory mem)
6964 %{
6965 match(Set dst (LoadS mem));
6966
6967 ins_cost(125);
6968 format %{ "movswl $dst, $mem\t# short" %}
6969
6970 ins_encode %{
6971 __ movswl($dst$$Register, $mem$$Address);
6972 %}
6973
6974 ins_pipe(ialu_reg_mem);
6975 %}
6976
6977 // Load Short (16 bit signed) to Byte (8 bit signed)
6978 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
6979 match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
6980
6981 ins_cost(125);
6982 format %{ "movsbl $dst, $mem\t# short -> byte" %}
6983 ins_encode %{
6984 __ movsbl($dst$$Register, $mem$$Address);
6985 %}
6986 ins_pipe(ialu_reg_mem);
6987 %}
6988
6989 // Load Short (16 bit signed) into Long Register
6990 instruct loadS2L(rRegL dst, memory mem)
6991 %{
6992 match(Set dst (ConvI2L (LoadS mem)));
6993
6994 ins_cost(125);
6995 format %{ "movswq $dst, $mem\t# short -> long" %}
6996
6997 ins_encode %{
6998 __ movswq($dst$$Register, $mem$$Address);
6999 %}
7000
7001 ins_pipe(ialu_reg_mem);
7002 %}
7003
7004 // Load Unsigned Short/Char (16 bit UNsigned)
7005 instruct loadUS(rRegI dst, memory mem)
7006 %{
7007 match(Set dst (LoadUS mem));
7008
7009 ins_cost(125);
7010 format %{ "movzwl $dst, $mem\t# ushort/char" %}
7011
7012 ins_encode %{
7013 __ movzwl($dst$$Register, $mem$$Address);
7014 %}
7015
7016 ins_pipe(ialu_reg_mem);
7017 %}
7018
7019 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
7020 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
7021 match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
7022
7023 ins_cost(125);
7024 format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
7025 ins_encode %{
7026 __ movsbl($dst$$Register, $mem$$Address);
7027 %}
7028 ins_pipe(ialu_reg_mem);
7029 %}
7030
7031 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
7032 instruct loadUS2L(rRegL dst, memory mem)
7033 %{
7034 match(Set dst (ConvI2L (LoadUS mem)));
7035
7036 ins_cost(125);
7037 format %{ "movzwq $dst, $mem\t# ushort/char -> long" %}
7038
7039 ins_encode %{
7040 __ movzwq($dst$$Register, $mem$$Address);
7041 %}
7042
7043 ins_pipe(ialu_reg_mem);
7044 %}
7045
7046 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
7047 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
7048 match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
7049
7050 format %{ "movzbq $dst, $mem\t# ushort/char & 0xFF -> long" %}
7051 ins_encode %{
7052 __ movzbq($dst$$Register, $mem$$Address);
7053 %}
7054 ins_pipe(ialu_reg_mem);
7055 %}
7056
7057 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
7058 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
7059 match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
7060 effect(KILL cr);
7061
7062 format %{ "movzwq $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
7063 "andl $dst, right_n_bits($mask, 16)" %}
7064 ins_encode %{
7065 Register Rdst = $dst$$Register;
7066 __ movzwq(Rdst, $mem$$Address);
7067 __ andl(Rdst, $mask$$constant & right_n_bits(16));
7068 %}
7069 ins_pipe(ialu_reg_mem);
7070 %}
7071
7072 // Load Integer
7073 instruct loadI(rRegI dst, memory mem)
7074 %{
7075 match(Set dst (LoadI mem));
7076
7077 ins_cost(125);
7078 format %{ "movl $dst, $mem\t# int" %}
7079
7080 ins_encode %{
7081 __ movl($dst$$Register, $mem$$Address);
7082 %}
7083
7084 ins_pipe(ialu_reg_mem);
7085 %}
7086
7087 // Load Integer (32 bit signed) to Byte (8 bit signed)
7088 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
7089 match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
7090
7091 ins_cost(125);
7092 format %{ "movsbl $dst, $mem\t# int -> byte" %}
7093 ins_encode %{
7094 __ movsbl($dst$$Register, $mem$$Address);
7095 %}
7096 ins_pipe(ialu_reg_mem);
7097 %}
7098
7099 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
7100 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
7101 match(Set dst (AndI (LoadI mem) mask));
7102
7103 ins_cost(125);
7104 format %{ "movzbl $dst, $mem\t# int -> ubyte" %}
7105 ins_encode %{
7106 __ movzbl($dst$$Register, $mem$$Address);
7107 %}
7108 ins_pipe(ialu_reg_mem);
7109 %}
7110
7111 // Load Integer (32 bit signed) to Short (16 bit signed)
7112 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
7113 match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
7114
7115 ins_cost(125);
7116 format %{ "movswl $dst, $mem\t# int -> short" %}
7117 ins_encode %{
7118 __ movswl($dst$$Register, $mem$$Address);
7119 %}
7120 ins_pipe(ialu_reg_mem);
7121 %}
7122
7123 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
7124 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
7125 match(Set dst (AndI (LoadI mem) mask));
7126
7127 ins_cost(125);
7128 format %{ "movzwl $dst, $mem\t# int -> ushort/char" %}
7129 ins_encode %{
7130 __ movzwl($dst$$Register, $mem$$Address);
7131 %}
7132 ins_pipe(ialu_reg_mem);
7133 %}
7134
7135 // Load Integer into Long Register
7136 instruct loadI2L(rRegL dst, memory mem)
7137 %{
7138 match(Set dst (ConvI2L (LoadI mem)));
7139
7140 ins_cost(125);
7141 format %{ "movslq $dst, $mem\t# int -> long" %}
7142
7143 ins_encode %{
7144 __ movslq($dst$$Register, $mem$$Address);
7145 %}
7146
7147 ins_pipe(ialu_reg_mem);
7148 %}
7149
7150 // Load Integer with mask 0xFF into Long Register
7151 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
7152 match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
7153
7154 format %{ "movzbq $dst, $mem\t# int & 0xFF -> long" %}
7155 ins_encode %{
7156 __ movzbq($dst$$Register, $mem$$Address);
7157 %}
7158 ins_pipe(ialu_reg_mem);
7159 %}
7160
7161 // Load Integer with mask 0xFFFF into Long Register
7162 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
7163 match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
7164
7165 format %{ "movzwq $dst, $mem\t# int & 0xFFFF -> long" %}
7166 ins_encode %{
7167 __ movzwq($dst$$Register, $mem$$Address);
7168 %}
7169 ins_pipe(ialu_reg_mem);
7170 %}
7171
7172 // Load Integer with a 31-bit mask into Long Register
7173 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
7174 match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
7175 effect(KILL cr);
7176
7177 format %{ "movl $dst, $mem\t# int & 31-bit mask -> long\n\t"
7178 "andl $dst, $mask" %}
7179 ins_encode %{
7180 Register Rdst = $dst$$Register;
7181 __ movl(Rdst, $mem$$Address);
7182 __ andl(Rdst, $mask$$constant);
7183 %}
7184 ins_pipe(ialu_reg_mem);
7185 %}
7186
7187 // Load Unsigned Integer into Long Register
7188 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
7189 %{
7190 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7191
7192 ins_cost(125);
7193 format %{ "movl $dst, $mem\t# uint -> long" %}
7194
7195 ins_encode %{
7196 __ movl($dst$$Register, $mem$$Address);
7197 %}
7198
7199 ins_pipe(ialu_reg_mem);
7200 %}
7201
7202 // Load Long
7203 instruct loadL(rRegL dst, memory mem)
7204 %{
7205 match(Set dst (LoadL mem));
7206
7207 ins_cost(125);
7208 format %{ "movq $dst, $mem\t# long" %}
7209
7210 ins_encode %{
7211 __ movq($dst$$Register, $mem$$Address);
7212 %}
7213
7214 ins_pipe(ialu_reg_mem); // XXX
7215 %}
7216
7217 // Load Range
7218 instruct loadRange(rRegI dst, memory mem)
7219 %{
7220 match(Set dst (LoadRange mem));
7221
7222 ins_cost(125); // XXX
7223 format %{ "movl $dst, $mem\t# range" %}
7224 ins_encode %{
7225 __ movl($dst$$Register, $mem$$Address);
7226 %}
7227 ins_pipe(ialu_reg_mem);
7228 %}
7229
7230 // Load Pointer
7231 instruct loadP(rRegP dst, memory mem)
7232 %{
7233 match(Set dst (LoadP mem));
7234 predicate(n->as_Load()->barrier_data() == 0);
7235
7236 ins_cost(125); // XXX
7237 format %{ "movq $dst, $mem\t# ptr" %}
7238 ins_encode %{
7239 __ movq($dst$$Register, $mem$$Address);
7240 %}
7241 ins_pipe(ialu_reg_mem); // XXX
7242 %}
7243
7244 // Load Compressed Pointer
7245 instruct loadN(rRegN dst, memory mem)
7246 %{
7247 predicate(n->as_Load()->barrier_data() == 0);
7248 match(Set dst (LoadN mem));
7249
7250 ins_cost(125); // XXX
7251 format %{ "movl $dst, $mem\t# compressed ptr" %}
7252 ins_encode %{
7253 __ movl($dst$$Register, $mem$$Address);
7254 %}
7255 ins_pipe(ialu_reg_mem); // XXX
7256 %}
7257
7258
7259 // Load Klass Pointer
7260 instruct loadKlass(rRegP dst, memory mem)
7261 %{
7262 match(Set dst (LoadKlass mem));
7263
7264 ins_cost(125); // XXX
7265 format %{ "movq $dst, $mem\t# class" %}
7266 ins_encode %{
7267 __ movq($dst$$Register, $mem$$Address);
7268 %}
7269 ins_pipe(ialu_reg_mem); // XXX
7270 %}
7271
7272 // Load narrow Klass Pointer
7273 instruct loadNKlass(rRegN dst, memory mem)
7274 %{
7275 predicate(!UseCompactObjectHeaders);
7276 match(Set dst (LoadNKlass mem));
7277
7278 ins_cost(125); // XXX
7279 format %{ "movl $dst, $mem\t# compressed klass ptr" %}
7280 ins_encode %{
7281 __ movl($dst$$Register, $mem$$Address);
7282 %}
7283 ins_pipe(ialu_reg_mem); // XXX
7284 %}
7285
7286 instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr)
7287 %{
7288 predicate(UseCompactObjectHeaders);
7289 match(Set dst (LoadNKlass mem));
7290 effect(KILL cr);
7291 ins_cost(125);
7292 format %{
7293 "movl $dst, $mem\t# compressed klass ptr, shifted\n\t"
7294 "shrl $dst, markWord::klass_shift_at_offset"
7295 %}
7296 ins_encode %{
7297 __ movl($dst$$Register, $mem$$Address);
7298 __ shrl($dst$$Register, markWord::klass_shift_at_offset);
7299 %}
7300 ins_pipe(ialu_reg_mem);
7301 %}
7302
7303 // Load Float
7304 instruct loadF(regF dst, memory mem)
7305 %{
7306 match(Set dst (LoadF mem));
7307
7308 ins_cost(145); // XXX
7309 format %{ "movss $dst, $mem\t# float" %}
7310 ins_encode %{
7311 __ movflt($dst$$XMMRegister, $mem$$Address);
7312 %}
7313 ins_pipe(pipe_slow); // XXX
7314 %}
7315
7316 // Load Double
7317 instruct loadD_partial(regD dst, memory mem)
7318 %{
7319 predicate(!UseXmmLoadAndClearUpper);
7320 match(Set dst (LoadD mem));
7321
7322 ins_cost(145); // XXX
7323 format %{ "movlpd $dst, $mem\t# double" %}
7324 ins_encode %{
7325 __ movdbl($dst$$XMMRegister, $mem$$Address);
7326 %}
7327 ins_pipe(pipe_slow); // XXX
7328 %}
7329
7330 instruct loadD(regD dst, memory mem)
7331 %{
7332 predicate(UseXmmLoadAndClearUpper);
7333 match(Set dst (LoadD mem));
7334
7335 ins_cost(145); // XXX
7336 format %{ "movsd $dst, $mem\t# double" %}
7337 ins_encode %{
7338 __ movdbl($dst$$XMMRegister, $mem$$Address);
7339 %}
7340 ins_pipe(pipe_slow); // XXX
7341 %}
7342
7343 instruct loadAOTRCAddress(rRegP dst, immAOTRuntimeConstantsAddress con)
7344 %{
7345 match(Set dst con);
7346
7347 format %{ "leaq $dst, $con\t# AOT Runtime Constants Address" %}
7348
7349 ins_encode %{
7350 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
7351 %}
7352
7353 ins_pipe(ialu_reg_fat);
7354 %}
7355
7356 // min = java.lang.Math.min(float a, float b)
7357 // max = java.lang.Math.max(float a, float b)
7358 instruct minmaxF_reg_avx10_2(regF dst, regF a, regF b)
7359 %{
7360 predicate(VM_Version::supports_avx10_2() && !VLoopReductions::is_reduction(n));
7361 match(Set dst (MaxF a b));
7362 match(Set dst (MinF a b));
7363
7364 format %{ "minmaxF $dst, $a, $b" %}
7365 ins_encode %{
7366 int opcode = this->ideal_Opcode();
7367 __ sminmax_fp_avx10_2(opcode, T_FLOAT, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister);
7368 %}
7369 ins_pipe( pipe_slow );
7370 %}
7371
7372 instruct minmaxF_reduction_reg_avx10_2(regF dst, regF a, regF b, rRegI rtmp, rFlagsReg cr)
7373 %{
7374 predicate(VM_Version::supports_avx10_2() && VLoopReductions::is_reduction(n));
7375 match(Set dst (MaxF a b));
7376 match(Set dst (MinF a b));
7377 effect(USE a, USE b, TEMP rtmp, KILL cr);
7378
7379 format %{ "minmaxF_reduction $dst, $a, $b \t! using $rtmp as TEMP" %}
7380 ins_encode %{
7381 int opcode = this->ideal_Opcode();
7382 bool min = (opcode == Op_MinF) ? true : false;
7383 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7384 min, fp_prec_flt /*pt*/);
7385 %}
7386 ins_pipe( pipe_slow );
7387 %}
7388
7389 // min = java.lang.Math.min(float a, float b)
7390 // max = java.lang.Math.max(float a, float b)
7391 instruct minmaxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp)
7392 %{
7393 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
7394 match(Set dst (MaxF a b));
7395 match(Set dst (MinF a b));
7396 effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
7397
7398 format %{ "minmaxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
7399 ins_encode %{
7400 int opcode = this->ideal_Opcode();
7401 int param_opcode = (opcode == Op_MinF) ? Op_MinV : Op_MaxV;
7402 __ vminmax_fp(param_opcode, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister,
7403 $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
7404 %}
7405 ins_pipe( pipe_slow );
7406 %}
7407
7408 instruct minmaxF_reduction_reg(legRegF dst, legRegF a, legRegF b, rRegI rtmp, rFlagsReg cr)
7409 %{
7410 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
7411 match(Set dst (MaxF a b));
7412 match(Set dst (MinF a b));
7413 effect(USE a, USE b, TEMP rtmp, KILL cr);
7414
7415 format %{ "minmaxF_reduction $dst, $a, $b \t!using $rtmp as TEMP" %}
7416 ins_encode %{
7417 int opcode = this->ideal_Opcode();
7418 bool min = (opcode == Op_MinF) ? true : false;
7419 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7420 min, fp_prec_flt /*pt*/);
7421 %}
7422 ins_pipe( pipe_slow );
7423 %}
7424
7425 // min = java.lang.Math.min(double a, double b)
7426 // max = java.lang.Math.max(double a, double b)
7427 instruct minmaxD_reg_avx10_2(regD dst, regD a, regD b)
7428 %{
7429 predicate(VM_Version::supports_avx10_2() && !VLoopReductions::is_reduction(n));
7430 match(Set dst (MaxD a b));
7431 match(Set dst (MinD a b));
7432
7433 format %{ "minmaxD $dst, $a, $b" %}
7434 ins_encode %{
7435 int opcode = this->ideal_Opcode();
7436 __ sminmax_fp_avx10_2(opcode, T_DOUBLE, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister);
7437 %}
7438 ins_pipe( pipe_slow );
7439 %}
7440
7441 instruct minmaxD_reduction_reg_avx10_2(regD dst, regD a, regD b, rRegI rtmp, rFlagsReg cr)
7442 %{
7443 predicate(VM_Version::supports_avx10_2() && VLoopReductions::is_reduction(n));
7444 match(Set dst (MaxD a b));
7445 match(Set dst (MinD a b));
7446 effect(USE a, USE b, TEMP rtmp, KILL cr);
7447
7448 format %{ "minmaxD_reduction $dst, $a, $b \t! using $rtmp as TEMP" %}
7449 ins_encode %{
7450 int opcode = this->ideal_Opcode();
7451 bool min = (opcode == Op_MinD) ? true : false;
7452 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7453 min, fp_prec_dbl /*pt*/);
7454 %}
7455 ins_pipe( pipe_slow );
7456 %}
7457
7458 // min = java.lang.Math.min(double a, double b)
7459 // max = java.lang.Math.max(double a, double b)
7460 instruct minmaxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp)
7461 %{
7462 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
7463 match(Set dst (MaxD a b));
7464 match(Set dst (MinD a b));
7465 effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
7466
7467 format %{ "minmaxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
7468 ins_encode %{
7469 int opcode = this->ideal_Opcode();
7470 int param_opcode = (opcode == Op_MinD) ? Op_MinV : Op_MaxV;
7471 __ vminmax_fp(param_opcode, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister,
7472 $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
7473 %}
7474 ins_pipe( pipe_slow );
7475 %}
7476
7477 instruct minmaxD_reduction_reg(legRegD dst, legRegD a, legRegD b, rRegL rtmp, rFlagsReg cr)
7478 %{
7479 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
7480 match(Set dst (MaxD a b));
7481 match(Set dst (MinD a b));
7482 effect(USE a, USE b, TEMP rtmp, KILL cr);
7483
7484 format %{ "minmaxD_reduction $dst, $a, $b \t! using $rtmp as TEMP" %}
7485 ins_encode %{
7486 int opcode = this->ideal_Opcode();
7487 bool min = (opcode == Op_MinD) ? true : false;
7488 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7489 min, fp_prec_dbl /*pt*/);
7490 %}
7491 ins_pipe( pipe_slow );
7492 %}
7493
7494 // Load Effective Address
7495 instruct leaP8(rRegP dst, indOffset8 mem)
7496 %{
7497 match(Set dst mem);
7498
7499 ins_cost(110); // XXX
7500 format %{ "leaq $dst, $mem\t# ptr 8" %}
7501 ins_encode %{
7502 __ leaq($dst$$Register, $mem$$Address);
7503 %}
7504 ins_pipe(ialu_reg_reg_fat);
7505 %}
7506
7507 instruct leaP32(rRegP dst, indOffset32 mem)
7508 %{
7509 match(Set dst mem);
7510
7511 ins_cost(110);
7512 format %{ "leaq $dst, $mem\t# ptr 32" %}
7513 ins_encode %{
7514 __ leaq($dst$$Register, $mem$$Address);
7515 %}
7516 ins_pipe(ialu_reg_reg_fat);
7517 %}
7518
7519 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
7520 %{
7521 match(Set dst mem);
7522
7523 ins_cost(110);
7524 format %{ "leaq $dst, $mem\t# ptr idxoff" %}
7525 ins_encode %{
7526 __ leaq($dst$$Register, $mem$$Address);
7527 %}
7528 ins_pipe(ialu_reg_reg_fat);
7529 %}
7530
7531 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
7532 %{
7533 match(Set dst mem);
7534
7535 ins_cost(110);
7536 format %{ "leaq $dst, $mem\t# ptr idxscale" %}
7537 ins_encode %{
7538 __ leaq($dst$$Register, $mem$$Address);
7539 %}
7540 ins_pipe(ialu_reg_reg_fat);
7541 %}
7542
7543 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
7544 %{
7545 match(Set dst mem);
7546
7547 ins_cost(110);
7548 format %{ "leaq $dst, $mem\t# ptr idxscale" %}
7549 ins_encode %{
7550 __ leaq($dst$$Register, $mem$$Address);
7551 %}
7552 ins_pipe(ialu_reg_reg_fat);
7553 %}
7554
7555 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
7556 %{
7557 match(Set dst mem);
7558
7559 ins_cost(110);
7560 format %{ "leaq $dst, $mem\t# ptr idxscaleoff" %}
7561 ins_encode %{
7562 __ leaq($dst$$Register, $mem$$Address);
7563 %}
7564 ins_pipe(ialu_reg_reg_fat);
7565 %}
7566
7567 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
7568 %{
7569 match(Set dst mem);
7570
7571 ins_cost(110);
7572 format %{ "leaq $dst, $mem\t# ptr posidxoff" %}
7573 ins_encode %{
7574 __ leaq($dst$$Register, $mem$$Address);
7575 %}
7576 ins_pipe(ialu_reg_reg_fat);
7577 %}
7578
7579 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
7580 %{
7581 match(Set dst mem);
7582
7583 ins_cost(110);
7584 format %{ "leaq $dst, $mem\t# ptr posidxscaleoff" %}
7585 ins_encode %{
7586 __ leaq($dst$$Register, $mem$$Address);
7587 %}
7588 ins_pipe(ialu_reg_reg_fat);
7589 %}
7590
7591 // Load Effective Address which uses Narrow (32-bits) oop
7592 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
7593 %{
7594 predicate(UseCompressedOops && (CompressedOops::shift() != 0));
7595 match(Set dst mem);
7596
7597 ins_cost(110);
7598 format %{ "leaq $dst, $mem\t# ptr compressedoopoff32" %}
7599 ins_encode %{
7600 __ leaq($dst$$Register, $mem$$Address);
7601 %}
7602 ins_pipe(ialu_reg_reg_fat);
7603 %}
7604
7605 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
7606 %{
7607 predicate(CompressedOops::shift() == 0);
7608 match(Set dst mem);
7609
7610 ins_cost(110); // XXX
7611 format %{ "leaq $dst, $mem\t# ptr off8narrow" %}
7612 ins_encode %{
7613 __ leaq($dst$$Register, $mem$$Address);
7614 %}
7615 ins_pipe(ialu_reg_reg_fat);
7616 %}
7617
7618 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
7619 %{
7620 predicate(CompressedOops::shift() == 0);
7621 match(Set dst mem);
7622
7623 ins_cost(110);
7624 format %{ "leaq $dst, $mem\t# ptr off32narrow" %}
7625 ins_encode %{
7626 __ leaq($dst$$Register, $mem$$Address);
7627 %}
7628 ins_pipe(ialu_reg_reg_fat);
7629 %}
7630
7631 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
7632 %{
7633 predicate(CompressedOops::shift() == 0);
7634 match(Set dst mem);
7635
7636 ins_cost(110);
7637 format %{ "leaq $dst, $mem\t# ptr idxoffnarrow" %}
7638 ins_encode %{
7639 __ leaq($dst$$Register, $mem$$Address);
7640 %}
7641 ins_pipe(ialu_reg_reg_fat);
7642 %}
7643
7644 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
7645 %{
7646 predicate(CompressedOops::shift() == 0);
7647 match(Set dst mem);
7648
7649 ins_cost(110);
7650 format %{ "leaq $dst, $mem\t# ptr idxscalenarrow" %}
7651 ins_encode %{
7652 __ leaq($dst$$Register, $mem$$Address);
7653 %}
7654 ins_pipe(ialu_reg_reg_fat);
7655 %}
7656
7657 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
7658 %{
7659 predicate(CompressedOops::shift() == 0);
7660 match(Set dst mem);
7661
7662 ins_cost(110);
7663 format %{ "leaq $dst, $mem\t# ptr idxscaleoffnarrow" %}
7664 ins_encode %{
7665 __ leaq($dst$$Register, $mem$$Address);
7666 %}
7667 ins_pipe(ialu_reg_reg_fat);
7668 %}
7669
7670 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
7671 %{
7672 predicate(CompressedOops::shift() == 0);
7673 match(Set dst mem);
7674
7675 ins_cost(110);
7676 format %{ "leaq $dst, $mem\t# ptr posidxoffnarrow" %}
7677 ins_encode %{
7678 __ leaq($dst$$Register, $mem$$Address);
7679 %}
7680 ins_pipe(ialu_reg_reg_fat);
7681 %}
7682
7683 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
7684 %{
7685 predicate(CompressedOops::shift() == 0);
7686 match(Set dst mem);
7687
7688 ins_cost(110);
7689 format %{ "leaq $dst, $mem\t# ptr posidxscaleoffnarrow" %}
7690 ins_encode %{
7691 __ leaq($dst$$Register, $mem$$Address);
7692 %}
7693 ins_pipe(ialu_reg_reg_fat);
7694 %}
7695
7696 instruct loadConI(rRegI dst, immI src)
7697 %{
7698 match(Set dst src);
7699
7700 format %{ "movl $dst, $src\t# int" %}
7701 ins_encode %{
7702 __ movl($dst$$Register, $src$$constant);
7703 %}
7704 ins_pipe(ialu_reg_fat); // XXX
7705 %}
7706
7707 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
7708 %{
7709 match(Set dst src);
7710 effect(KILL cr);
7711
7712 ins_cost(50);
7713 format %{ "xorl $dst, $dst\t# int" %}
7714 ins_encode %{
7715 __ xorl($dst$$Register, $dst$$Register);
7716 %}
7717 ins_pipe(ialu_reg);
7718 %}
7719
7720 instruct loadConL(rRegL dst, immL src)
7721 %{
7722 match(Set dst src);
7723
7724 ins_cost(150);
7725 format %{ "movq $dst, $src\t# long" %}
7726 ins_encode %{
7727 __ mov64($dst$$Register, $src$$constant);
7728 %}
7729 ins_pipe(ialu_reg);
7730 %}
7731
7732 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
7733 %{
7734 match(Set dst src);
7735 effect(KILL cr);
7736
7737 ins_cost(50);
7738 format %{ "xorl $dst, $dst\t# long" %}
7739 ins_encode %{
7740 __ xorl($dst$$Register, $dst$$Register);
7741 %}
7742 ins_pipe(ialu_reg); // XXX
7743 %}
7744
7745 instruct loadConUL32(rRegL dst, immUL32 src)
7746 %{
7747 match(Set dst src);
7748
7749 ins_cost(60);
7750 format %{ "movl $dst, $src\t# long (unsigned 32-bit)" %}
7751 ins_encode %{
7752 __ movl($dst$$Register, $src$$constant);
7753 %}
7754 ins_pipe(ialu_reg);
7755 %}
7756
7757 instruct loadConL32(rRegL dst, immL32 src)
7758 %{
7759 match(Set dst src);
7760
7761 ins_cost(70);
7762 format %{ "movq $dst, $src\t# long (32-bit)" %}
7763 ins_encode %{
7764 __ movq($dst$$Register, $src$$constant);
7765 %}
7766 ins_pipe(ialu_reg);
7767 %}
7768
7769 instruct loadConP(rRegP dst, immP con) %{
7770 match(Set dst con);
7771
7772 format %{ "movq $dst, $con\t# ptr" %}
7773 ins_encode %{
7774 __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
7775 %}
7776 ins_pipe(ialu_reg_fat); // XXX
7777 %}
7778
7779 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
7780 %{
7781 match(Set dst src);
7782 effect(KILL cr);
7783
7784 ins_cost(50);
7785 format %{ "xorl $dst, $dst\t# ptr" %}
7786 ins_encode %{
7787 __ xorl($dst$$Register, $dst$$Register);
7788 %}
7789 ins_pipe(ialu_reg);
7790 %}
7791
7792 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
7793 %{
7794 match(Set dst src);
7795 effect(KILL cr);
7796
7797 ins_cost(60);
7798 format %{ "movl $dst, $src\t# ptr (positive 32-bit)" %}
7799 ins_encode %{
7800 __ movl($dst$$Register, $src$$constant);
7801 %}
7802 ins_pipe(ialu_reg);
7803 %}
7804
7805 instruct loadConF(regF dst, immF con) %{
7806 match(Set dst con);
7807 ins_cost(125);
7808 format %{ "movss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
7809 ins_encode %{
7810 __ movflt($dst$$XMMRegister, $constantaddress($con));
7811 %}
7812 ins_pipe(pipe_slow);
7813 %}
7814
7815 instruct loadConH(regF dst, immH con) %{
7816 match(Set dst con);
7817 ins_cost(125);
7818 format %{ "movss $dst, [$constantaddress]\t# load from constant table: halffloat=$con" %}
7819 ins_encode %{
7820 __ movflt($dst$$XMMRegister, $constantaddress($con));
7821 %}
7822 ins_pipe(pipe_slow);
7823 %}
7824
7825 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
7826 match(Set dst src);
7827 effect(KILL cr);
7828 format %{ "xorq $dst, $src\t# compressed null pointer" %}
7829 ins_encode %{
7830 __ xorq($dst$$Register, $dst$$Register);
7831 %}
7832 ins_pipe(ialu_reg);
7833 %}
7834
7835 instruct loadConN(rRegN dst, immN src) %{
7836 match(Set dst src);
7837
7838 ins_cost(125);
7839 format %{ "movl $dst, $src\t# compressed ptr" %}
7840 ins_encode %{
7841 address con = (address)$src$$constant;
7842 if (con == nullptr) {
7843 ShouldNotReachHere();
7844 } else {
7845 __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
7846 }
7847 %}
7848 ins_pipe(ialu_reg_fat); // XXX
7849 %}
7850
7851 instruct loadConNKlass(rRegN dst, immNKlass src) %{
7852 match(Set dst src);
7853
7854 ins_cost(125);
7855 format %{ "movl $dst, $src\t# compressed klass ptr" %}
7856 ins_encode %{
7857 address con = (address)$src$$constant;
7858 if (con == nullptr) {
7859 ShouldNotReachHere();
7860 } else {
7861 __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
7862 }
7863 %}
7864 ins_pipe(ialu_reg_fat); // XXX
7865 %}
7866
7867 instruct loadConF0(regF dst, immF0 src)
7868 %{
7869 match(Set dst src);
7870 ins_cost(100);
7871
7872 format %{ "xorps $dst, $dst\t# float 0.0" %}
7873 ins_encode %{
7874 __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
7875 %}
7876 ins_pipe(pipe_slow);
7877 %}
7878
7879 // Use the same format since predicate() can not be used here.
7880 instruct loadConD(regD dst, immD con) %{
7881 match(Set dst con);
7882 ins_cost(125);
7883 format %{ "movsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
7884 ins_encode %{
7885 __ movdbl($dst$$XMMRegister, $constantaddress($con));
7886 %}
7887 ins_pipe(pipe_slow);
7888 %}
7889
7890 instruct loadConD0(regD dst, immD0 src)
7891 %{
7892 match(Set dst src);
7893 ins_cost(100);
7894
7895 format %{ "xorpd $dst, $dst\t# double 0.0" %}
7896 ins_encode %{
7897 __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
7898 %}
7899 ins_pipe(pipe_slow);
7900 %}
7901
7902 instruct loadSSI(rRegI dst, stackSlotI src)
7903 %{
7904 match(Set dst src);
7905
7906 ins_cost(125);
7907 format %{ "movl $dst, $src\t# int stk" %}
7908 ins_encode %{
7909 __ movl($dst$$Register, $src$$Address);
7910 %}
7911 ins_pipe(ialu_reg_mem);
7912 %}
7913
7914 instruct loadSSL(rRegL dst, stackSlotL src)
7915 %{
7916 match(Set dst src);
7917
7918 ins_cost(125);
7919 format %{ "movq $dst, $src\t# long stk" %}
7920 ins_encode %{
7921 __ movq($dst$$Register, $src$$Address);
7922 %}
7923 ins_pipe(ialu_reg_mem);
7924 %}
7925
7926 instruct loadSSP(rRegP dst, stackSlotP src)
7927 %{
7928 match(Set dst src);
7929
7930 ins_cost(125);
7931 format %{ "movq $dst, $src\t# ptr stk" %}
7932 ins_encode %{
7933 __ movq($dst$$Register, $src$$Address);
7934 %}
7935 ins_pipe(ialu_reg_mem);
7936 %}
7937
7938 instruct loadSSF(regF dst, stackSlotF src)
7939 %{
7940 match(Set dst src);
7941
7942 ins_cost(125);
7943 format %{ "movss $dst, $src\t# float stk" %}
7944 ins_encode %{
7945 __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
7946 %}
7947 ins_pipe(pipe_slow); // XXX
7948 %}
7949
7950 // Use the same format since predicate() can not be used here.
7951 instruct loadSSD(regD dst, stackSlotD src)
7952 %{
7953 match(Set dst src);
7954
7955 ins_cost(125);
7956 format %{ "movsd $dst, $src\t# double stk" %}
7957 ins_encode %{
7958 __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
7959 %}
7960 ins_pipe(pipe_slow); // XXX
7961 %}
7962
7963 // Prefetch instructions for allocation.
7964 // Must be safe to execute with invalid address (cannot fault).
7965
7966 instruct prefetchAlloc( memory mem ) %{
7967 predicate(AllocatePrefetchInstr==3);
7968 match(PrefetchAllocation mem);
7969 ins_cost(125);
7970
7971 format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
7972 ins_encode %{
7973 __ prefetchw($mem$$Address);
7974 %}
7975 ins_pipe(ialu_mem);
7976 %}
7977
7978 instruct prefetchAllocNTA( memory mem ) %{
7979 predicate(AllocatePrefetchInstr==0);
7980 match(PrefetchAllocation mem);
7981 ins_cost(125);
7982
7983 format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
7984 ins_encode %{
7985 __ prefetchnta($mem$$Address);
7986 %}
7987 ins_pipe(ialu_mem);
7988 %}
7989
7990 instruct prefetchAllocT0( memory mem ) %{
7991 predicate(AllocatePrefetchInstr==1);
7992 match(PrefetchAllocation mem);
7993 ins_cost(125);
7994
7995 format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
7996 ins_encode %{
7997 __ prefetcht0($mem$$Address);
7998 %}
7999 ins_pipe(ialu_mem);
8000 %}
8001
8002 instruct prefetchAllocT2( memory mem ) %{
8003 predicate(AllocatePrefetchInstr==2);
8004 match(PrefetchAllocation mem);
8005 ins_cost(125);
8006
8007 format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
8008 ins_encode %{
8009 __ prefetcht2($mem$$Address);
8010 %}
8011 ins_pipe(ialu_mem);
8012 %}
8013
8014 //----------Store Instructions-------------------------------------------------
8015
8016 // Store Byte
8017 instruct storeB(memory mem, rRegI src)
8018 %{
8019 match(Set mem (StoreB mem src));
8020
8021 ins_cost(125); // XXX
8022 format %{ "movb $mem, $src\t# byte" %}
8023 ins_encode %{
8024 __ movb($mem$$Address, $src$$Register);
8025 %}
8026 ins_pipe(ialu_mem_reg);
8027 %}
8028
8029 // Store Char/Short
8030 instruct storeC(memory mem, rRegI src)
8031 %{
8032 match(Set mem (StoreC mem src));
8033
8034 ins_cost(125); // XXX
8035 format %{ "movw $mem, $src\t# char/short" %}
8036 ins_encode %{
8037 __ movw($mem$$Address, $src$$Register);
8038 %}
8039 ins_pipe(ialu_mem_reg);
8040 %}
8041
8042 // Store Integer
8043 instruct storeI(memory mem, rRegI src)
8044 %{
8045 match(Set mem (StoreI mem src));
8046
8047 ins_cost(125); // XXX
8048 format %{ "movl $mem, $src\t# int" %}
8049 ins_encode %{
8050 __ movl($mem$$Address, $src$$Register);
8051 %}
8052 ins_pipe(ialu_mem_reg);
8053 %}
8054
8055 // Store Long
8056 instruct storeL(memory mem, rRegL src)
8057 %{
8058 match(Set mem (StoreL mem src));
8059
8060 ins_cost(125); // XXX
8061 format %{ "movq $mem, $src\t# long" %}
8062 ins_encode %{
8063 __ movq($mem$$Address, $src$$Register);
8064 %}
8065 ins_pipe(ialu_mem_reg); // XXX
8066 %}
8067
8068 // Store Pointer
8069 instruct storeP(memory mem, any_RegP src)
8070 %{
8071 predicate(n->as_Store()->barrier_data() == 0);
8072 match(Set mem (StoreP mem src));
8073
8074 ins_cost(125); // XXX
8075 format %{ "movq $mem, $src\t# ptr" %}
8076 ins_encode %{
8077 __ movq($mem$$Address, $src$$Register);
8078 %}
8079 ins_pipe(ialu_mem_reg);
8080 %}
8081
8082 instruct storeImmP0(memory mem, immP0 zero)
8083 %{
8084 predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
8085 match(Set mem (StoreP mem zero));
8086
8087 ins_cost(125); // XXX
8088 format %{ "movq $mem, R12\t# ptr (R12_heapbase==0)" %}
8089 ins_encode %{
8090 __ movq($mem$$Address, r12);
8091 %}
8092 ins_pipe(ialu_mem_reg);
8093 %}
8094
8095 // Store Null Pointer, mark word, or other simple pointer constant.
8096 instruct storeImmP(memory mem, immP31 src)
8097 %{
8098 predicate(n->as_Store()->barrier_data() == 0);
8099 match(Set mem (StoreP mem src));
8100
8101 ins_cost(150); // XXX
8102 format %{ "movq $mem, $src\t# ptr" %}
8103 ins_encode %{
8104 __ movq($mem$$Address, $src$$constant);
8105 %}
8106 ins_pipe(ialu_mem_imm);
8107 %}
8108
8109 // Store Compressed Pointer
8110 instruct storeN(memory mem, rRegN src)
8111 %{
8112 predicate(n->as_Store()->barrier_data() == 0);
8113 match(Set mem (StoreN mem src));
8114
8115 ins_cost(125); // XXX
8116 format %{ "movl $mem, $src\t# compressed ptr" %}
8117 ins_encode %{
8118 __ movl($mem$$Address, $src$$Register);
8119 %}
8120 ins_pipe(ialu_mem_reg);
8121 %}
8122
8123 instruct storeNKlass(memory mem, rRegN src)
8124 %{
8125 match(Set mem (StoreNKlass mem src));
8126
8127 ins_cost(125); // XXX
8128 format %{ "movl $mem, $src\t# compressed klass ptr" %}
8129 ins_encode %{
8130 __ movl($mem$$Address, $src$$Register);
8131 %}
8132 ins_pipe(ialu_mem_reg);
8133 %}
8134
8135 instruct storeImmN0(memory mem, immN0 zero)
8136 %{
8137 predicate(CompressedOops::base() == nullptr && n->as_Store()->barrier_data() == 0);
8138 match(Set mem (StoreN mem zero));
8139
8140 ins_cost(125); // XXX
8141 format %{ "movl $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
8142 ins_encode %{
8143 __ movl($mem$$Address, r12);
8144 %}
8145 ins_pipe(ialu_mem_reg);
8146 %}
8147
8148 instruct storeImmN(memory mem, immN src)
8149 %{
8150 predicate(n->as_Store()->barrier_data() == 0);
8151 match(Set mem (StoreN mem src));
8152
8153 ins_cost(150); // XXX
8154 format %{ "movl $mem, $src\t# compressed ptr" %}
8155 ins_encode %{
8156 address con = (address)$src$$constant;
8157 if (con == nullptr) {
8158 __ movl($mem$$Address, 0);
8159 } else {
8160 __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
8161 }
8162 %}
8163 ins_pipe(ialu_mem_imm);
8164 %}
8165
8166 instruct storeImmNKlass(memory mem, immNKlass src)
8167 %{
8168 match(Set mem (StoreNKlass mem src));
8169
8170 ins_cost(150); // XXX
8171 format %{ "movl $mem, $src\t# compressed klass ptr" %}
8172 ins_encode %{
8173 __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
8174 %}
8175 ins_pipe(ialu_mem_imm);
8176 %}
8177
8178 // Store Integer Immediate
8179 instruct storeImmI0(memory mem, immI_0 zero)
8180 %{
8181 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8182 match(Set mem (StoreI mem zero));
8183
8184 ins_cost(125); // XXX
8185 format %{ "movl $mem, R12\t# int (R12_heapbase==0)" %}
8186 ins_encode %{
8187 __ movl($mem$$Address, r12);
8188 %}
8189 ins_pipe(ialu_mem_reg);
8190 %}
8191
8192 instruct storeImmI(memory mem, immI src)
8193 %{
8194 match(Set mem (StoreI mem src));
8195
8196 ins_cost(150);
8197 format %{ "movl $mem, $src\t# int" %}
8198 ins_encode %{
8199 __ movl($mem$$Address, $src$$constant);
8200 %}
8201 ins_pipe(ialu_mem_imm);
8202 %}
8203
8204 // Store Long Immediate
8205 instruct storeImmL0(memory mem, immL0 zero)
8206 %{
8207 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8208 match(Set mem (StoreL mem zero));
8209
8210 ins_cost(125); // XXX
8211 format %{ "movq $mem, R12\t# long (R12_heapbase==0)" %}
8212 ins_encode %{
8213 __ movq($mem$$Address, r12);
8214 %}
8215 ins_pipe(ialu_mem_reg);
8216 %}
8217
8218 instruct storeImmL(memory mem, immL32 src)
8219 %{
8220 match(Set mem (StoreL mem src));
8221
8222 ins_cost(150);
8223 format %{ "movq $mem, $src\t# long" %}
8224 ins_encode %{
8225 __ movq($mem$$Address, $src$$constant);
8226 %}
8227 ins_pipe(ialu_mem_imm);
8228 %}
8229
8230 // Store Short/Char Immediate
8231 instruct storeImmC0(memory mem, immI_0 zero)
8232 %{
8233 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8234 match(Set mem (StoreC mem zero));
8235
8236 ins_cost(125); // XXX
8237 format %{ "movw $mem, R12\t# short/char (R12_heapbase==0)" %}
8238 ins_encode %{
8239 __ movw($mem$$Address, r12);
8240 %}
8241 ins_pipe(ialu_mem_reg);
8242 %}
8243
8244 instruct storeImmI16(memory mem, immI16 src)
8245 %{
8246 predicate(UseStoreImmI16);
8247 match(Set mem (StoreC mem src));
8248
8249 ins_cost(150);
8250 format %{ "movw $mem, $src\t# short/char" %}
8251 ins_encode %{
8252 __ movw($mem$$Address, $src$$constant);
8253 %}
8254 ins_pipe(ialu_mem_imm);
8255 %}
8256
8257 // Store Byte Immediate
8258 instruct storeImmB0(memory mem, immI_0 zero)
8259 %{
8260 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8261 match(Set mem (StoreB mem zero));
8262
8263 ins_cost(125); // XXX
8264 format %{ "movb $mem, R12\t# short/char (R12_heapbase==0)" %}
8265 ins_encode %{
8266 __ movb($mem$$Address, r12);
8267 %}
8268 ins_pipe(ialu_mem_reg);
8269 %}
8270
8271 instruct storeImmB(memory mem, immI8 src)
8272 %{
8273 match(Set mem (StoreB mem src));
8274
8275 ins_cost(150); // XXX
8276 format %{ "movb $mem, $src\t# byte" %}
8277 ins_encode %{
8278 __ movb($mem$$Address, $src$$constant);
8279 %}
8280 ins_pipe(ialu_mem_imm);
8281 %}
8282
8283 // Store Float
8284 instruct storeF(memory mem, regF src)
8285 %{
8286 match(Set mem (StoreF mem src));
8287
8288 ins_cost(95); // XXX
8289 format %{ "movss $mem, $src\t# float" %}
8290 ins_encode %{
8291 __ movflt($mem$$Address, $src$$XMMRegister);
8292 %}
8293 ins_pipe(pipe_slow); // XXX
8294 %}
8295
8296 // Store immediate Float value (it is faster than store from XMM register)
8297 instruct storeF0(memory mem, immF0 zero)
8298 %{
8299 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8300 match(Set mem (StoreF mem zero));
8301
8302 ins_cost(25); // XXX
8303 format %{ "movl $mem, R12\t# float 0. (R12_heapbase==0)" %}
8304 ins_encode %{
8305 __ movl($mem$$Address, r12);
8306 %}
8307 ins_pipe(ialu_mem_reg);
8308 %}
8309
8310 instruct storeF_imm(memory mem, immF src)
8311 %{
8312 match(Set mem (StoreF mem src));
8313
8314 ins_cost(50);
8315 format %{ "movl $mem, $src\t# float" %}
8316 ins_encode %{
8317 __ movl($mem$$Address, jint_cast($src$$constant));
8318 %}
8319 ins_pipe(ialu_mem_imm);
8320 %}
8321
8322 // Store Double
8323 instruct storeD(memory mem, regD src)
8324 %{
8325 match(Set mem (StoreD mem src));
8326
8327 ins_cost(95); // XXX
8328 format %{ "movsd $mem, $src\t# double" %}
8329 ins_encode %{
8330 __ movdbl($mem$$Address, $src$$XMMRegister);
8331 %}
8332 ins_pipe(pipe_slow); // XXX
8333 %}
8334
8335 // Store immediate double 0.0 (it is faster than store from XMM register)
8336 instruct storeD0_imm(memory mem, immD0 src)
8337 %{
8338 predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
8339 match(Set mem (StoreD mem src));
8340
8341 ins_cost(50);
8342 format %{ "movq $mem, $src\t# double 0." %}
8343 ins_encode %{
8344 __ movq($mem$$Address, $src$$constant);
8345 %}
8346 ins_pipe(ialu_mem_imm);
8347 %}
8348
8349 instruct storeD0(memory mem, immD0 zero)
8350 %{
8351 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8352 match(Set mem (StoreD mem zero));
8353
8354 ins_cost(25); // XXX
8355 format %{ "movq $mem, R12\t# double 0. (R12_heapbase==0)" %}
8356 ins_encode %{
8357 __ movq($mem$$Address, r12);
8358 %}
8359 ins_pipe(ialu_mem_reg);
8360 %}
8361
8362 instruct storeSSI(stackSlotI dst, rRegI src)
8363 %{
8364 match(Set dst src);
8365
8366 ins_cost(100);
8367 format %{ "movl $dst, $src\t# int stk" %}
8368 ins_encode %{
8369 __ movl($dst$$Address, $src$$Register);
8370 %}
8371 ins_pipe( ialu_mem_reg );
8372 %}
8373
8374 instruct storeSSL(stackSlotL dst, rRegL src)
8375 %{
8376 match(Set dst src);
8377
8378 ins_cost(100);
8379 format %{ "movq $dst, $src\t# long stk" %}
8380 ins_encode %{
8381 __ movq($dst$$Address, $src$$Register);
8382 %}
8383 ins_pipe(ialu_mem_reg);
8384 %}
8385
8386 instruct storeSSP(stackSlotP dst, rRegP src)
8387 %{
8388 match(Set dst src);
8389
8390 ins_cost(100);
8391 format %{ "movq $dst, $src\t# ptr stk" %}
8392 ins_encode %{
8393 __ movq($dst$$Address, $src$$Register);
8394 %}
8395 ins_pipe(ialu_mem_reg);
8396 %}
8397
8398 instruct storeSSF(stackSlotF dst, regF src)
8399 %{
8400 match(Set dst src);
8401
8402 ins_cost(95); // XXX
8403 format %{ "movss $dst, $src\t# float stk" %}
8404 ins_encode %{
8405 __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
8406 %}
8407 ins_pipe(pipe_slow); // XXX
8408 %}
8409
8410 instruct storeSSD(stackSlotD dst, regD src)
8411 %{
8412 match(Set dst src);
8413
8414 ins_cost(95); // XXX
8415 format %{ "movsd $dst, $src\t# double stk" %}
8416 ins_encode %{
8417 __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
8418 %}
8419 ins_pipe(pipe_slow); // XXX
8420 %}
8421
8422 instruct cacheWB(indirect addr)
8423 %{
8424 predicate(VM_Version::supports_data_cache_line_flush());
8425 match(CacheWB addr);
8426
8427 ins_cost(100);
8428 format %{"cache wb $addr" %}
8429 ins_encode %{
8430 assert($addr->index_position() < 0, "should be");
8431 assert($addr$$disp == 0, "should be");
8432 __ cache_wb(Address($addr$$base$$Register, 0));
8433 %}
8434 ins_pipe(pipe_slow); // XXX
8435 %}
8436
8437 instruct cacheWBPreSync()
8438 %{
8439 predicate(VM_Version::supports_data_cache_line_flush());
8440 match(CacheWBPreSync);
8441
8442 ins_cost(100);
8443 format %{"cache wb presync" %}
8444 ins_encode %{
8445 __ cache_wbsync(true);
8446 %}
8447 ins_pipe(pipe_slow); // XXX
8448 %}
8449
8450 instruct cacheWBPostSync()
8451 %{
8452 predicate(VM_Version::supports_data_cache_line_flush());
8453 match(CacheWBPostSync);
8454
8455 ins_cost(100);
8456 format %{"cache wb postsync" %}
8457 ins_encode %{
8458 __ cache_wbsync(false);
8459 %}
8460 ins_pipe(pipe_slow); // XXX
8461 %}
8462
8463 //----------BSWAP Instructions-------------------------------------------------
8464 instruct bytes_reverse_int(rRegI dst) %{
8465 match(Set dst (ReverseBytesI dst));
8466
8467 format %{ "bswapl $dst" %}
8468 ins_encode %{
8469 __ bswapl($dst$$Register);
8470 %}
8471 ins_pipe( ialu_reg );
8472 %}
8473
8474 instruct bytes_reverse_long(rRegL dst) %{
8475 match(Set dst (ReverseBytesL dst));
8476
8477 format %{ "bswapq $dst" %}
8478 ins_encode %{
8479 __ bswapq($dst$$Register);
8480 %}
8481 ins_pipe( ialu_reg);
8482 %}
8483
8484 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
8485 match(Set dst (ReverseBytesUS dst));
8486 effect(KILL cr);
8487
8488 format %{ "bswapl $dst\n\t"
8489 "shrl $dst,16\n\t" %}
8490 ins_encode %{
8491 __ bswapl($dst$$Register);
8492 __ shrl($dst$$Register, 16);
8493 %}
8494 ins_pipe( ialu_reg );
8495 %}
8496
8497 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
8498 match(Set dst (ReverseBytesS dst));
8499 effect(KILL cr);
8500
8501 format %{ "bswapl $dst\n\t"
8502 "sar $dst,16\n\t" %}
8503 ins_encode %{
8504 __ bswapl($dst$$Register);
8505 __ sarl($dst$$Register, 16);
8506 %}
8507 ins_pipe( ialu_reg );
8508 %}
8509
8510 //---------- Zeros Count Instructions ------------------------------------------
8511
8512 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
8513 predicate(UseCountLeadingZerosInstruction);
8514 match(Set dst (CountLeadingZerosI src));
8515 effect(KILL cr);
8516
8517 format %{ "lzcntl $dst, $src\t# count leading zeros (int)" %}
8518 ins_encode %{
8519 __ lzcntl($dst$$Register, $src$$Register);
8520 %}
8521 ins_pipe(ialu_reg);
8522 %}
8523
8524 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
8525 predicate(UseCountLeadingZerosInstruction);
8526 match(Set dst (CountLeadingZerosI (LoadI src)));
8527 effect(KILL cr);
8528 ins_cost(175);
8529 format %{ "lzcntl $dst, $src\t# count leading zeros (int)" %}
8530 ins_encode %{
8531 __ lzcntl($dst$$Register, $src$$Address);
8532 %}
8533 ins_pipe(ialu_reg_mem);
8534 %}
8535
8536 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
8537 predicate(!UseCountLeadingZerosInstruction);
8538 match(Set dst (CountLeadingZerosI src));
8539 effect(KILL cr);
8540
8541 format %{ "bsrl $dst, $src\t# count leading zeros (int)\n\t"
8542 "jnz skip\n\t"
8543 "movl $dst, -1\n"
8544 "skip:\n\t"
8545 "negl $dst\n\t"
8546 "addl $dst, 31" %}
8547 ins_encode %{
8548 Register Rdst = $dst$$Register;
8549 Register Rsrc = $src$$Register;
8550 Label skip;
8551 __ bsrl(Rdst, Rsrc);
8552 __ jccb(Assembler::notZero, skip);
8553 __ movl(Rdst, -1);
8554 __ bind(skip);
8555 __ negl(Rdst);
8556 __ addl(Rdst, BitsPerInt - 1);
8557 %}
8558 ins_pipe(ialu_reg);
8559 %}
8560
8561 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
8562 predicate(UseCountLeadingZerosInstruction);
8563 match(Set dst (CountLeadingZerosL src));
8564 effect(KILL cr);
8565
8566 format %{ "lzcntq $dst, $src\t# count leading zeros (long)" %}
8567 ins_encode %{
8568 __ lzcntq($dst$$Register, $src$$Register);
8569 %}
8570 ins_pipe(ialu_reg);
8571 %}
8572
8573 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
8574 predicate(UseCountLeadingZerosInstruction);
8575 match(Set dst (CountLeadingZerosL (LoadL src)));
8576 effect(KILL cr);
8577 ins_cost(175);
8578 format %{ "lzcntq $dst, $src\t# count leading zeros (long)" %}
8579 ins_encode %{
8580 __ lzcntq($dst$$Register, $src$$Address);
8581 %}
8582 ins_pipe(ialu_reg_mem);
8583 %}
8584
8585 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
8586 predicate(!UseCountLeadingZerosInstruction);
8587 match(Set dst (CountLeadingZerosL src));
8588 effect(KILL cr);
8589
8590 format %{ "bsrq $dst, $src\t# count leading zeros (long)\n\t"
8591 "jnz skip\n\t"
8592 "movl $dst, -1\n"
8593 "skip:\n\t"
8594 "negl $dst\n\t"
8595 "addl $dst, 63" %}
8596 ins_encode %{
8597 Register Rdst = $dst$$Register;
8598 Register Rsrc = $src$$Register;
8599 Label skip;
8600 __ bsrq(Rdst, Rsrc);
8601 __ jccb(Assembler::notZero, skip);
8602 __ movl(Rdst, -1);
8603 __ bind(skip);
8604 __ negl(Rdst);
8605 __ addl(Rdst, BitsPerLong - 1);
8606 %}
8607 ins_pipe(ialu_reg);
8608 %}
8609
8610 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
8611 predicate(UseCountTrailingZerosInstruction);
8612 match(Set dst (CountTrailingZerosI src));
8613 effect(KILL cr);
8614
8615 format %{ "tzcntl $dst, $src\t# count trailing zeros (int)" %}
8616 ins_encode %{
8617 __ tzcntl($dst$$Register, $src$$Register);
8618 %}
8619 ins_pipe(ialu_reg);
8620 %}
8621
8622 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
8623 predicate(UseCountTrailingZerosInstruction);
8624 match(Set dst (CountTrailingZerosI (LoadI src)));
8625 effect(KILL cr);
8626 ins_cost(175);
8627 format %{ "tzcntl $dst, $src\t# count trailing zeros (int)" %}
8628 ins_encode %{
8629 __ tzcntl($dst$$Register, $src$$Address);
8630 %}
8631 ins_pipe(ialu_reg_mem);
8632 %}
8633
8634 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
8635 predicate(!UseCountTrailingZerosInstruction);
8636 match(Set dst (CountTrailingZerosI src));
8637 effect(KILL cr);
8638
8639 format %{ "bsfl $dst, $src\t# count trailing zeros (int)\n\t"
8640 "jnz done\n\t"
8641 "movl $dst, 32\n"
8642 "done:" %}
8643 ins_encode %{
8644 Register Rdst = $dst$$Register;
8645 Label done;
8646 __ bsfl(Rdst, $src$$Register);
8647 __ jccb(Assembler::notZero, done);
8648 __ movl(Rdst, BitsPerInt);
8649 __ bind(done);
8650 %}
8651 ins_pipe(ialu_reg);
8652 %}
8653
8654 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
8655 predicate(UseCountTrailingZerosInstruction);
8656 match(Set dst (CountTrailingZerosL src));
8657 effect(KILL cr);
8658
8659 format %{ "tzcntq $dst, $src\t# count trailing zeros (long)" %}
8660 ins_encode %{
8661 __ tzcntq($dst$$Register, $src$$Register);
8662 %}
8663 ins_pipe(ialu_reg);
8664 %}
8665
8666 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
8667 predicate(UseCountTrailingZerosInstruction);
8668 match(Set dst (CountTrailingZerosL (LoadL src)));
8669 effect(KILL cr);
8670 ins_cost(175);
8671 format %{ "tzcntq $dst, $src\t# count trailing zeros (long)" %}
8672 ins_encode %{
8673 __ tzcntq($dst$$Register, $src$$Address);
8674 %}
8675 ins_pipe(ialu_reg_mem);
8676 %}
8677
8678 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
8679 predicate(!UseCountTrailingZerosInstruction);
8680 match(Set dst (CountTrailingZerosL src));
8681 effect(KILL cr);
8682
8683 format %{ "bsfq $dst, $src\t# count trailing zeros (long)\n\t"
8684 "jnz done\n\t"
8685 "movl $dst, 64\n"
8686 "done:" %}
8687 ins_encode %{
8688 Register Rdst = $dst$$Register;
8689 Label done;
8690 __ bsfq(Rdst, $src$$Register);
8691 __ jccb(Assembler::notZero, done);
8692 __ movl(Rdst, BitsPerLong);
8693 __ bind(done);
8694 %}
8695 ins_pipe(ialu_reg);
8696 %}
8697
8698 //--------------- Reverse Operation Instructions ----------------
8699 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
8700 predicate(!VM_Version::supports_gfni());
8701 match(Set dst (ReverseI src));
8702 effect(TEMP dst, TEMP rtmp, KILL cr);
8703 format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
8704 ins_encode %{
8705 __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
8706 %}
8707 ins_pipe( ialu_reg );
8708 %}
8709
8710 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, vlRegF xtmp1, vlRegF xtmp2, rRegL rtmp, rFlagsReg cr) %{
8711 predicate(VM_Version::supports_gfni());
8712 match(Set dst (ReverseI src));
8713 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
8714 format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
8715 ins_encode %{
8716 __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
8717 %}
8718 ins_pipe( ialu_reg );
8719 %}
8720
8721 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
8722 predicate(!VM_Version::supports_gfni());
8723 match(Set dst (ReverseL src));
8724 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
8725 format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
8726 ins_encode %{
8727 __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
8728 %}
8729 ins_pipe( ialu_reg );
8730 %}
8731
8732 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, vlRegD xtmp1, vlRegD xtmp2, rRegL rtmp, rFlagsReg cr) %{
8733 predicate(VM_Version::supports_gfni());
8734 match(Set dst (ReverseL src));
8735 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
8736 format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
8737 ins_encode %{
8738 __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
8739 %}
8740 ins_pipe( ialu_reg );
8741 %}
8742
8743 //---------- Population Count Instructions -------------------------------------
8744
8745 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
8746 predicate(UsePopCountInstruction);
8747 match(Set dst (PopCountI src));
8748 effect(KILL cr);
8749
8750 format %{ "popcnt $dst, $src" %}
8751 ins_encode %{
8752 __ popcntl($dst$$Register, $src$$Register);
8753 %}
8754 ins_pipe(ialu_reg);
8755 %}
8756
8757 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
8758 predicate(UsePopCountInstruction);
8759 match(Set dst (PopCountI (LoadI mem)));
8760 effect(KILL cr);
8761
8762 format %{ "popcnt $dst, $mem" %}
8763 ins_encode %{
8764 __ popcntl($dst$$Register, $mem$$Address);
8765 %}
8766 ins_pipe(ialu_reg);
8767 %}
8768
8769 // Note: Long.bitCount(long) returns an int.
8770 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
8771 predicate(UsePopCountInstruction);
8772 match(Set dst (PopCountL src));
8773 effect(KILL cr);
8774
8775 format %{ "popcnt $dst, $src" %}
8776 ins_encode %{
8777 __ popcntq($dst$$Register, $src$$Register);
8778 %}
8779 ins_pipe(ialu_reg);
8780 %}
8781
8782 // Note: Long.bitCount(long) returns an int.
8783 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
8784 predicate(UsePopCountInstruction);
8785 match(Set dst (PopCountL (LoadL mem)));
8786 effect(KILL cr);
8787
8788 format %{ "popcnt $dst, $mem" %}
8789 ins_encode %{
8790 __ popcntq($dst$$Register, $mem$$Address);
8791 %}
8792 ins_pipe(ialu_reg);
8793 %}
8794
8795
8796 //----------MemBar Instructions-----------------------------------------------
8797 // Memory barrier flavors
8798
8799 instruct membar_acquire()
8800 %{
8801 match(MemBarAcquire);
8802 match(LoadFence);
8803 ins_cost(0);
8804
8805 size(0);
8806 format %{ "MEMBAR-acquire ! (empty encoding)" %}
8807 ins_encode();
8808 ins_pipe(empty);
8809 %}
8810
8811 instruct membar_acquire_lock()
8812 %{
8813 match(MemBarAcquireLock);
8814 ins_cost(0);
8815
8816 size(0);
8817 format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
8818 ins_encode();
8819 ins_pipe(empty);
8820 %}
8821
8822 instruct membar_release()
8823 %{
8824 match(MemBarRelease);
8825 match(StoreFence);
8826 ins_cost(0);
8827
8828 size(0);
8829 format %{ "MEMBAR-release ! (empty encoding)" %}
8830 ins_encode();
8831 ins_pipe(empty);
8832 %}
8833
8834 instruct membar_release_lock()
8835 %{
8836 match(MemBarReleaseLock);
8837 ins_cost(0);
8838
8839 size(0);
8840 format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
8841 ins_encode();
8842 ins_pipe(empty);
8843 %}
8844
8845 instruct membar_storeload(rFlagsReg cr) %{
8846 match(MemBarStoreLoad);
8847 effect(KILL cr);
8848 ins_cost(400);
8849
8850 format %{
8851 $$template
8852 $$emit$$"lock addl [rsp + #0], 0\t! membar_storeload"
8853 %}
8854 ins_encode %{
8855 __ membar(Assembler::StoreLoad);
8856 %}
8857 ins_pipe(pipe_slow);
8858 %}
8859
8860 instruct membar_volatile(rFlagsReg cr) %{
8861 match(MemBarVolatile);
8862 effect(KILL cr);
8863 ins_cost(400);
8864
8865 format %{
8866 $$template
8867 $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
8868 %}
8869 ins_encode %{
8870 __ membar(Assembler::StoreLoad);
8871 %}
8872 ins_pipe(pipe_slow);
8873 %}
8874
8875 instruct unnecessary_membar_volatile()
8876 %{
8877 match(MemBarVolatile);
8878 predicate(Matcher::post_store_load_barrier(n));
8879 ins_cost(0);
8880
8881 size(0);
8882 format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
8883 ins_encode();
8884 ins_pipe(empty);
8885 %}
8886
8887 instruct membar_full(rFlagsReg cr) %{
8888 match(MemBarFull);
8889 effect(KILL cr);
8890 ins_cost(400);
8891
8892 format %{
8893 $$template
8894 $$emit$$"lock addl [rsp + #0], 0\t! membar_full"
8895 %}
8896 ins_encode %{
8897 __ membar(Assembler::StoreLoad);
8898 %}
8899 ins_pipe(pipe_slow);
8900 %}
8901
8902 instruct membar_storestore() %{
8903 match(MemBarStoreStore);
8904 match(StoreStoreFence);
8905 ins_cost(0);
8906
8907 size(0);
8908 format %{ "MEMBAR-storestore (empty encoding)" %}
8909 ins_encode( );
8910 ins_pipe(empty);
8911 %}
8912
8913 //----------Move Instructions--------------------------------------------------
8914
8915 instruct castX2P(rRegP dst, rRegL src)
8916 %{
8917 match(Set dst (CastX2P src));
8918
8919 format %{ "movq $dst, $src\t# long->ptr" %}
8920 ins_encode %{
8921 if ($dst$$reg != $src$$reg) {
8922 __ movptr($dst$$Register, $src$$Register);
8923 }
8924 %}
8925 ins_pipe(ialu_reg_reg); // XXX
8926 %}
8927
8928 instruct castP2X(rRegL dst, rRegP src)
8929 %{
8930 match(Set dst (CastP2X src));
8931
8932 format %{ "movq $dst, $src\t# ptr -> long" %}
8933 ins_encode %{
8934 if ($dst$$reg != $src$$reg) {
8935 __ movptr($dst$$Register, $src$$Register);
8936 }
8937 %}
8938 ins_pipe(ialu_reg_reg); // XXX
8939 %}
8940
8941 // Convert oop into int for vectors alignment masking
8942 instruct convP2I(rRegI dst, rRegP src)
8943 %{
8944 match(Set dst (ConvL2I (CastP2X src)));
8945
8946 format %{ "movl $dst, $src\t# ptr -> int" %}
8947 ins_encode %{
8948 __ movl($dst$$Register, $src$$Register);
8949 %}
8950 ins_pipe(ialu_reg_reg); // XXX
8951 %}
8952
8953 // Convert compressed oop into int for vectors alignment masking
8954 // in case of 32bit oops (heap < 4Gb).
8955 instruct convN2I(rRegI dst, rRegN src)
8956 %{
8957 predicate(CompressedOops::shift() == 0);
8958 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8959
8960 format %{ "movl $dst, $src\t# compressed ptr -> int" %}
8961 ins_encode %{
8962 __ movl($dst$$Register, $src$$Register);
8963 %}
8964 ins_pipe(ialu_reg_reg); // XXX
8965 %}
8966
8967 // Convert oop pointer into compressed form
8968 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
8969 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8970 match(Set dst (EncodeP src));
8971 effect(KILL cr);
8972 format %{ "encode_heap_oop $dst,$src" %}
8973 ins_encode %{
8974 Register s = $src$$Register;
8975 Register d = $dst$$Register;
8976 if (s != d) {
8977 __ movq(d, s);
8978 }
8979 __ encode_heap_oop(d);
8980 %}
8981 ins_pipe(ialu_reg_long);
8982 %}
8983
8984 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
8985 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8986 match(Set dst (EncodeP src));
8987 effect(KILL cr);
8988 format %{ "encode_heap_oop_not_null $dst,$src" %}
8989 ins_encode %{
8990 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8991 %}
8992 ins_pipe(ialu_reg_long);
8993 %}
8994
8995 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
8996 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8997 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8998 match(Set dst (DecodeN src));
8999 effect(KILL cr);
9000 format %{ "decode_heap_oop $dst,$src" %}
9001 ins_encode %{
9002 Register s = $src$$Register;
9003 Register d = $dst$$Register;
9004 if (s != d) {
9005 __ movq(d, s);
9006 }
9007 __ decode_heap_oop(d);
9008 %}
9009 ins_pipe(ialu_reg_long);
9010 %}
9011
9012 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
9013 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
9014 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
9015 match(Set dst (DecodeN src));
9016 effect(KILL cr);
9017 format %{ "decode_heap_oop_not_null $dst,$src" %}
9018 ins_encode %{
9019 Register s = $src$$Register;
9020 Register d = $dst$$Register;
9021 if (s != d) {
9022 __ decode_heap_oop_not_null(d, s);
9023 } else {
9024 __ decode_heap_oop_not_null(d);
9025 }
9026 %}
9027 ins_pipe(ialu_reg_long);
9028 %}
9029
9030 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
9031 match(Set dst (EncodePKlass src));
9032 effect(TEMP dst, KILL cr);
9033 format %{ "encode_and_move_klass_not_null $dst,$src" %}
9034 ins_encode %{
9035 __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
9036 %}
9037 ins_pipe(ialu_reg_long);
9038 %}
9039
9040 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
9041 match(Set dst (DecodeNKlass src));
9042 effect(TEMP dst, KILL cr);
9043 format %{ "decode_and_move_klass_not_null $dst,$src" %}
9044 ins_encode %{
9045 __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
9046 %}
9047 ins_pipe(ialu_reg_long);
9048 %}
9049
9050 //----------Conditional Move---------------------------------------------------
9051 // Jump
9052 // dummy instruction for generating temp registers
9053 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
9054 match(Jump (LShiftL switch_val shift));
9055 ins_cost(350);
9056 predicate(false);
9057 effect(TEMP dest);
9058
9059 format %{ "leaq $dest, [$constantaddress]\n\t"
9060 "jmp [$dest + $switch_val << $shift]\n\t" %}
9061 ins_encode %{
9062 // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
9063 // to do that and the compiler is using that register as one it can allocate.
9064 // So we build it all by hand.
9065 // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
9066 // ArrayAddress dispatch(table, index);
9067 Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
9068 __ lea($dest$$Register, $constantaddress);
9069 __ jmp(dispatch);
9070 %}
9071 ins_pipe(pipe_jmp);
9072 %}
9073
9074 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
9075 match(Jump (AddL (LShiftL switch_val shift) offset));
9076 ins_cost(350);
9077 effect(TEMP dest);
9078
9079 format %{ "leaq $dest, [$constantaddress]\n\t"
9080 "jmp [$dest + $switch_val << $shift + $offset]\n\t" %}
9081 ins_encode %{
9082 // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
9083 // to do that and the compiler is using that register as one it can allocate.
9084 // So we build it all by hand.
9085 // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
9086 // ArrayAddress dispatch(table, index);
9087 Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
9088 __ lea($dest$$Register, $constantaddress);
9089 __ jmp(dispatch);
9090 %}
9091 ins_pipe(pipe_jmp);
9092 %}
9093
9094 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
9095 match(Jump switch_val);
9096 ins_cost(350);
9097 effect(TEMP dest);
9098
9099 format %{ "leaq $dest, [$constantaddress]\n\t"
9100 "jmp [$dest + $switch_val]\n\t" %}
9101 ins_encode %{
9102 // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
9103 // to do that and the compiler is using that register as one it can allocate.
9104 // So we build it all by hand.
9105 // Address index(noreg, switch_reg, Address::times_1);
9106 // ArrayAddress dispatch(table, index);
9107 Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
9108 __ lea($dest$$Register, $constantaddress);
9109 __ jmp(dispatch);
9110 %}
9111 ins_pipe(pipe_jmp);
9112 %}
9113
9114 // Conditional move
9115 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
9116 %{
9117 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9118 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9119
9120 ins_cost(100); // XXX
9121 format %{ "setbn$cop $dst\t# signed, int" %}
9122 ins_encode %{
9123 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9124 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9125 %}
9126 ins_pipe(ialu_reg);
9127 %}
9128
9129 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
9130 %{
9131 predicate(!UseAPX);
9132 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9133
9134 ins_cost(200); // XXX
9135 format %{ "cmovl$cop $dst, $src\t# signed, int" %}
9136 ins_encode %{
9137 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9138 %}
9139 ins_pipe(pipe_cmov_reg);
9140 %}
9141
9142 instruct cmovI_reg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr, cmpOp cop)
9143 %{
9144 predicate(UseAPX);
9145 match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
9146
9147 ins_cost(200);
9148 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
9149 ins_encode %{
9150 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9151 %}
9152 ins_pipe(pipe_cmov_reg);
9153 %}
9154
9155 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
9156 %{
9157 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9158 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9159
9160 ins_cost(100); // XXX
9161 format %{ "setbn$cop $dst\t# unsigned, int" %}
9162 ins_encode %{
9163 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9164 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9165 %}
9166 ins_pipe(ialu_reg);
9167 %}
9168
9169 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
9170 predicate(!UseAPX);
9171 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9172
9173 ins_cost(200); // XXX
9174 format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
9175 ins_encode %{
9176 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9177 %}
9178 ins_pipe(pipe_cmov_reg);
9179 %}
9180
9181 instruct cmovI_regU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, rRegI src2) %{
9182 predicate(UseAPX);
9183 match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
9184
9185 ins_cost(200);
9186 format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
9187 ins_encode %{
9188 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9189 %}
9190 ins_pipe(pipe_cmov_reg);
9191 %}
9192
9193 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
9194 %{
9195 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9196 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9197
9198 ins_cost(100); // XXX
9199 format %{ "setbn$cop $dst\t# unsigned, int" %}
9200 ins_encode %{
9201 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9202 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9203 %}
9204 ins_pipe(ialu_reg);
9205 %}
9206
9207 instruct cmovI_imm_01UCFE(rRegI dst, immI_1 src, rFlagsRegUCFE cr, cmpOpUCFE cop)
9208 %{
9209 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9210 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9211
9212 ins_cost(100); // XXX
9213 format %{ "setbn$cop $dst\t# signed, unsigned, int" %}
9214 ins_encode %{
9215 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9216 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9217 %}
9218 ins_pipe(ialu_reg);
9219 %}
9220
9221 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
9222 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9223
9224 ins_cost(200);
9225 expand %{
9226 cmovI_regU(cop, cr, dst, src);
9227 %}
9228 %}
9229
9230 instruct cmovI_regUCFE_ndd(rRegI dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegI src1, rRegI src2) %{
9231 match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
9232
9233 ins_cost(200);
9234 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, unsigned, int ndd" %}
9235 ins_encode %{
9236 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9237 %}
9238 ins_pipe(pipe_cmov_reg);
9239 %}
9240
9241 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
9242 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9243 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9244
9245 ins_cost(200); // XXX
9246 format %{ "cmovpl $dst, $src\n\t"
9247 "cmovnel $dst, $src" %}
9248 ins_encode %{
9249 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9250 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9251 %}
9252 ins_pipe(pipe_cmov_reg);
9253 %}
9254
9255 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9256 // inputs of the CMove
9257 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
9258 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9259 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9260 effect(TEMP dst);
9261
9262 ins_cost(200); // XXX
9263 format %{ "cmovpl $dst, $src\n\t"
9264 "cmovnel $dst, $src" %}
9265 ins_encode %{
9266 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9267 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9268 %}
9269 ins_pipe(pipe_cmov_reg);
9270 %}
9271
9272 // Conditional move
9273 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
9274 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9275
9276 ins_cost(250); // XXX
9277 format %{ "cmovl$cop $dst, $src\t# signed, int" %}
9278 ins_encode %{
9279 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9280 %}
9281 ins_pipe(pipe_cmov_mem);
9282 %}
9283
9284 // Conditional move
9285 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
9286 %{
9287 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9288
9289 ins_cost(250); // XXX
9290 format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
9291 ins_encode %{
9292 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9293 %}
9294 ins_pipe(pipe_cmov_mem);
9295 %}
9296
9297 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
9298 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9299
9300 ins_cost(250);
9301 expand %{
9302 cmovI_memU(cop, cr, dst, src);
9303 %}
9304 %}
9305
9306 instruct cmovI_memUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, rRegI dst, memory src) %{
9307 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9308
9309 ins_cost(250); // XXX
9310 format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
9311 ins_encode %{
9312 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9313 %}
9314 ins_pipe(pipe_cmov_mem);
9315 %}
9316
9317 // Conditional move
9318 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
9319 %{
9320 predicate(!UseAPX);
9321 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9322
9323 ins_cost(200); // XXX
9324 format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
9325 ins_encode %{
9326 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9327 %}
9328 ins_pipe(pipe_cmov_reg);
9329 %}
9330
9331 // Conditional move ndd
9332 instruct cmovN_reg_ndd(rRegN dst, rRegN src1, rRegN src2, rFlagsReg cr, cmpOp cop)
9333 %{
9334 predicate(UseAPX);
9335 match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
9336
9337 ins_cost(200);
9338 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, compressed ptr ndd" %}
9339 ins_encode %{
9340 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9341 %}
9342 ins_pipe(pipe_cmov_reg);
9343 %}
9344
9345 // Conditional move
9346 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
9347 %{
9348 predicate(!UseAPX);
9349 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9350
9351 ins_cost(200); // XXX
9352 format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
9353 ins_encode %{
9354 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9355 %}
9356 ins_pipe(pipe_cmov_reg);
9357 %}
9358
9359 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
9360 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9361
9362 ins_cost(200);
9363 expand %{
9364 cmovN_regU(cop, cr, dst, src);
9365 %}
9366 %}
9367
9368 // Conditional move ndd
9369 instruct cmovN_regU_ndd(rRegN dst, cmpOpU cop, rFlagsRegU cr, rRegN src1, rRegN src2)
9370 %{
9371 predicate(UseAPX);
9372 match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
9373
9374 ins_cost(200);
9375 format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
9376 ins_encode %{
9377 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9378 %}
9379 ins_pipe(pipe_cmov_reg);
9380 %}
9381
9382 instruct cmovN_regUCFE_ndd(rRegN dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegN src1, rRegN src2) %{
9383 match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
9384
9385 ins_cost(200);
9386 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, unsigned, compressed ptr ndd" %}
9387 ins_encode %{
9388 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9389 %}
9390 ins_pipe(pipe_cmov_reg);
9391 %}
9392
9393 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
9394 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9395 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9396
9397 ins_cost(200); // XXX
9398 format %{ "cmovpl $dst, $src\n\t"
9399 "cmovnel $dst, $src" %}
9400 ins_encode %{
9401 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9402 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9403 %}
9404 ins_pipe(pipe_cmov_reg);
9405 %}
9406
9407 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9408 // inputs of the CMove
9409 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
9410 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9411 match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
9412
9413 ins_cost(200); // XXX
9414 format %{ "cmovpl $dst, $src\n\t"
9415 "cmovnel $dst, $src" %}
9416 ins_encode %{
9417 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9418 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9419 %}
9420 ins_pipe(pipe_cmov_reg);
9421 %}
9422
9423 // Conditional move
9424 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
9425 %{
9426 predicate(!UseAPX);
9427 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9428
9429 ins_cost(200); // XXX
9430 format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
9431 ins_encode %{
9432 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9433 %}
9434 ins_pipe(pipe_cmov_reg); // XXX
9435 %}
9436
9437 // Conditional move ndd
9438 instruct cmovP_reg_ndd(rRegP dst, rRegP src1, rRegP src2, rFlagsReg cr, cmpOp cop)
9439 %{
9440 predicate(UseAPX);
9441 match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
9442
9443 ins_cost(200);
9444 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, ptr ndd" %}
9445 ins_encode %{
9446 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9447 %}
9448 ins_pipe(pipe_cmov_reg);
9449 %}
9450
9451 // Conditional move
9452 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
9453 %{
9454 predicate(!UseAPX);
9455 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9456
9457 ins_cost(200); // XXX
9458 format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
9459 ins_encode %{
9460 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9461 %}
9462 ins_pipe(pipe_cmov_reg); // XXX
9463 %}
9464
9465 // Conditional move ndd
9466 instruct cmovP_regU_ndd(rRegP dst, cmpOpU cop, rFlagsRegU cr, rRegP src1, rRegP src2)
9467 %{
9468 predicate(UseAPX);
9469 match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
9470
9471 ins_cost(200);
9472 format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
9473 ins_encode %{
9474 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9475 %}
9476 ins_pipe(pipe_cmov_reg);
9477 %}
9478
9479 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
9480 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9481
9482 ins_cost(200);
9483 expand %{
9484 cmovP_regU(cop, cr, dst, src);
9485 %}
9486 %}
9487
9488 instruct cmovP_regUCFE_ndd(rRegP dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegP src1, rRegP src2) %{
9489 match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
9490
9491 ins_cost(200);
9492 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, unsigned, ptr ndd" %}
9493 ins_encode %{
9494 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9495 %}
9496 ins_pipe(pipe_cmov_reg);
9497 %}
9498
9499 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
9500 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9501 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9502
9503 ins_cost(200); // XXX
9504 format %{ "cmovpq $dst, $src\n\t"
9505 "cmovneq $dst, $src" %}
9506 ins_encode %{
9507 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9508 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9509 %}
9510 ins_pipe(pipe_cmov_reg);
9511 %}
9512
9513 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9514 // inputs of the CMove
9515 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
9516 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9517 match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
9518
9519 ins_cost(200); // XXX
9520 format %{ "cmovpq $dst, $src\n\t"
9521 "cmovneq $dst, $src" %}
9522 ins_encode %{
9523 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9524 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9525 %}
9526 ins_pipe(pipe_cmov_reg);
9527 %}
9528
9529 instruct cmovL_imm_01(rRegL dst, immL1 src, rFlagsReg cr, cmpOp cop)
9530 %{
9531 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9532 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9533
9534 ins_cost(100); // XXX
9535 format %{ "setbn$cop $dst\t# signed, long" %}
9536 ins_encode %{
9537 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9538 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9539 %}
9540 ins_pipe(ialu_reg);
9541 %}
9542
9543 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
9544 %{
9545 predicate(!UseAPX);
9546 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9547
9548 ins_cost(200); // XXX
9549 format %{ "cmovq$cop $dst, $src\t# signed, long" %}
9550 ins_encode %{
9551 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9552 %}
9553 ins_pipe(pipe_cmov_reg); // XXX
9554 %}
9555
9556 instruct cmovL_reg_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, rRegL src2)
9557 %{
9558 predicate(UseAPX);
9559 match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
9560
9561 ins_cost(200);
9562 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
9563 ins_encode %{
9564 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9565 %}
9566 ins_pipe(pipe_cmov_reg);
9567 %}
9568
9569 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
9570 %{
9571 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9572
9573 ins_cost(200); // XXX
9574 format %{ "cmovq$cop $dst, $src\t# signed, long" %}
9575 ins_encode %{
9576 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9577 %}
9578 ins_pipe(pipe_cmov_mem); // XXX
9579 %}
9580
9581 instruct cmovL_imm_01U(rRegL dst, immL1 src, rFlagsRegU cr, cmpOpU cop)
9582 %{
9583 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9584 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9585
9586 ins_cost(100); // XXX
9587 format %{ "setbn$cop $dst\t# unsigned, long" %}
9588 ins_encode %{
9589 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9590 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9591 %}
9592 ins_pipe(ialu_reg);
9593 %}
9594
9595 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
9596 %{
9597 predicate(!UseAPX);
9598 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9599
9600 ins_cost(200); // XXX
9601 format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
9602 ins_encode %{
9603 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9604 %}
9605 ins_pipe(pipe_cmov_reg); // XXX
9606 %}
9607
9608 instruct cmovL_regU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, rRegL src2)
9609 %{
9610 predicate(UseAPX);
9611 match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
9612
9613 ins_cost(200);
9614 format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
9615 ins_encode %{
9616 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9617 %}
9618 ins_pipe(pipe_cmov_reg);
9619 %}
9620
9621 instruct cmovL_imm_01UCF(rRegL dst, immL1 src, rFlagsRegUCF cr, cmpOpUCF cop)
9622 %{
9623 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9624 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9625
9626 ins_cost(100); // XXX
9627 format %{ "setbn$cop $dst\t# unsigned, long" %}
9628 ins_encode %{
9629 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9630 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9631 %}
9632 ins_pipe(ialu_reg);
9633 %}
9634
9635 instruct cmovL_imm_01UCFE(rRegL dst, immL1 src, rFlagsRegUCFE cr, cmpOpUCFE cop)
9636 %{
9637 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9638 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9639
9640 ins_cost(100); // XXX
9641 format %{ "setbn$cop $dst\t# signed, unsigned, long" %}
9642 ins_encode %{
9643 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9644 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9645 %}
9646 ins_pipe(ialu_reg);
9647 %}
9648
9649 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
9650 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9651
9652 ins_cost(200);
9653 expand %{
9654 cmovL_regU(cop, cr, dst, src);
9655 %}
9656 %}
9657
9658 instruct cmovL_regUCFE_ndd(rRegL dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegL src1, rRegL src2)
9659 %{
9660 match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
9661
9662 ins_cost(200);
9663 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, unsigned, long ndd" %}
9664 ins_encode %{
9665 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9666 %}
9667 ins_pipe(pipe_cmov_reg);
9668 %}
9669
9670 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
9671 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9672 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9673
9674 ins_cost(200); // XXX
9675 format %{ "cmovpq $dst, $src\n\t"
9676 "cmovneq $dst, $src" %}
9677 ins_encode %{
9678 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9679 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9680 %}
9681 ins_pipe(pipe_cmov_reg);
9682 %}
9683
9684 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9685 // inputs of the CMove
9686 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
9687 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9688 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9689
9690 ins_cost(200); // XXX
9691 format %{ "cmovpq $dst, $src\n\t"
9692 "cmovneq $dst, $src" %}
9693 ins_encode %{
9694 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9695 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9696 %}
9697 ins_pipe(pipe_cmov_reg);
9698 %}
9699
9700 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
9701 %{
9702 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9703
9704 ins_cost(200); // XXX
9705 format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
9706 ins_encode %{
9707 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9708 %}
9709 ins_pipe(pipe_cmov_mem); // XXX
9710 %}
9711
9712 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
9713 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9714
9715 ins_cost(200);
9716 expand %{
9717 cmovL_memU(cop, cr, dst, src);
9718 %}
9719 %}
9720
9721 instruct cmovL_memUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, rRegL dst, memory src) %{
9722 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9723
9724 ins_cost(200); // XXX
9725 format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
9726 ins_encode %{
9727 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9728 %}
9729 ins_pipe(pipe_cmov_mem); // XXX
9730 %}
9731
9732 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
9733 %{
9734 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9735
9736 ins_cost(200); // XXX
9737 format %{ "jn$cop skip\t# signed cmove float\n\t"
9738 "movss $dst, $src\n"
9739 "skip:" %}
9740 ins_encode %{
9741 Label Lskip;
9742 // Invert sense of branch from sense of CMOV
9743 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9744 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
9745 __ bind(Lskip);
9746 %}
9747 ins_pipe(pipe_slow);
9748 %}
9749
9750 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
9751 %{
9752 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9753
9754 ins_cost(200); // XXX
9755 format %{ "jn$cop skip\t# unsigned cmove float\n\t"
9756 "movss $dst, $src\n"
9757 "skip:" %}
9758 ins_encode %{
9759 Label Lskip;
9760 // Invert sense of branch from sense of CMOV
9761 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9762 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
9763 __ bind(Lskip);
9764 %}
9765 ins_pipe(pipe_slow);
9766 %}
9767
9768 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
9769 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9770
9771 ins_cost(200);
9772 expand %{
9773 cmovF_regU(cop, cr, dst, src);
9774 %}
9775 %}
9776
9777 instruct cmovF_regUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, regF dst, regF src)
9778 %{
9779 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9780
9781 ins_cost(200); // XXX
9782 format %{ "jn$cop skip\t# signed, unsigned cmove float\n\t"
9783 "movss $dst, $src\n"
9784 "skip:" %}
9785 ins_encode %{
9786 Label Lskip;
9787 // Invert sense of branch from sense of CMOV
9788 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9789 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
9790 __ bind(Lskip);
9791 %}
9792 ins_pipe(pipe_slow);
9793 %}
9794
9795 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
9796 %{
9797 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9798
9799 ins_cost(200); // XXX
9800 format %{ "jn$cop skip\t# signed cmove double\n\t"
9801 "movsd $dst, $src\n"
9802 "skip:" %}
9803 ins_encode %{
9804 Label Lskip;
9805 // Invert sense of branch from sense of CMOV
9806 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9807 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
9808 __ bind(Lskip);
9809 %}
9810 ins_pipe(pipe_slow);
9811 %}
9812
9813 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
9814 %{
9815 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9816
9817 ins_cost(200); // XXX
9818 format %{ "jn$cop skip\t# unsigned cmove double\n\t"
9819 "movsd $dst, $src\n"
9820 "skip:" %}
9821 ins_encode %{
9822 Label Lskip;
9823 // Invert sense of branch from sense of CMOV
9824 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9825 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
9826 __ bind(Lskip);
9827 %}
9828 ins_pipe(pipe_slow);
9829 %}
9830
9831 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
9832 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9833
9834 ins_cost(200);
9835 expand %{
9836 cmovD_regU(cop, cr, dst, src);
9837 %}
9838 %}
9839
9840 instruct cmovD_regUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, regD dst, regD src)
9841 %{
9842 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9843
9844 ins_cost(200); // XXX
9845 format %{ "jn$cop skip\t# signed, unsigned cmove double\n\t"
9846 "movsd $dst, $src\n"
9847 "skip:" %}
9848 ins_encode %{
9849 Label Lskip;
9850 // Invert sense of branch from sense of CMOV
9851 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9852 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
9853 __ bind(Lskip);
9854 %}
9855 ins_pipe(pipe_slow);
9856 %}
9857
9858 //----------Arithmetic Instructions--------------------------------------------
9859 //----------Addition Instructions----------------------------------------------
9860
9861 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9862 %{
9863 predicate(!UseAPX);
9864 match(Set dst (AddI dst src));
9865 effect(KILL cr);
9866 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9867 format %{ "addl $dst, $src\t# int" %}
9868 ins_encode %{
9869 __ addl($dst$$Register, $src$$Register);
9870 %}
9871 ins_pipe(ialu_reg_reg);
9872 %}
9873
9874 instruct addI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
9875 %{
9876 predicate(UseAPX);
9877 match(Set dst (AddI src1 src2));
9878 effect(KILL cr);
9879 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
9880
9881 format %{ "eaddl $dst, $src1, $src2\t# int ndd" %}
9882 ins_encode %{
9883 __ eaddl($dst$$Register, $src1$$Register, $src2$$Register, false);
9884 %}
9885 ins_pipe(ialu_reg_reg);
9886 %}
9887
9888 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9889 %{
9890 predicate(!UseAPX);
9891 match(Set dst (AddI dst src));
9892 effect(KILL cr);
9893 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9894
9895 format %{ "addl $dst, $src\t# int" %}
9896 ins_encode %{
9897 __ addl($dst$$Register, $src$$constant);
9898 %}
9899 ins_pipe( ialu_reg );
9900 %}
9901
9902 instruct addI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
9903 %{
9904 predicate(UseAPX);
9905 match(Set dst (AddI src1 src2));
9906 effect(KILL cr);
9907 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
9908
9909 format %{ "eaddl $dst, $src1, $src2\t# int ndd" %}
9910 ins_encode %{
9911 __ eaddl($dst$$Register, $src1$$Register, $src2$$constant, false);
9912 %}
9913 ins_pipe( ialu_reg );
9914 %}
9915
9916 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9917 %{
9918 match(Set dst (AddI dst (LoadI src)));
9919 effect(KILL cr);
9920 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9921
9922 ins_cost(150); // XXX
9923 format %{ "addl $dst, $src\t# int" %}
9924 ins_encode %{
9925 __ addl($dst$$Register, $src$$Address);
9926 %}
9927 ins_pipe(ialu_reg_mem);
9928 %}
9929
9930 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9931 %{
9932 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
9933 effect(KILL cr);
9934 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9935
9936 ins_cost(150); // XXX
9937 format %{ "addl $dst, $src\t# int" %}
9938 ins_encode %{
9939 __ addl($dst$$Address, $src$$Register);
9940 %}
9941 ins_pipe(ialu_mem_reg);
9942 %}
9943
9944 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
9945 %{
9946 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
9947 effect(KILL cr);
9948 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9949
9950
9951 ins_cost(125); // XXX
9952 format %{ "addl $dst, $src\t# int" %}
9953 ins_encode %{
9954 __ addl($dst$$Address, $src$$constant);
9955 %}
9956 ins_pipe(ialu_mem_imm);
9957 %}
9958
9959 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
9960 %{
9961 predicate(!UseAPX && UseIncDec);
9962 match(Set dst (AddI dst src));
9963 effect(KILL cr);
9964
9965 format %{ "incl $dst\t# int" %}
9966 ins_encode %{
9967 __ incrementl($dst$$Register);
9968 %}
9969 ins_pipe(ialu_reg);
9970 %}
9971
9972 instruct incI_rReg_ndd(rRegI dst, rRegI src, immI_1 val, rFlagsReg cr)
9973 %{
9974 predicate(UseAPX && UseIncDec);
9975 match(Set dst (AddI src val));
9976 effect(KILL cr);
9977 flag(PD::Flag_ndd_demotable_opr1);
9978
9979 format %{ "eincl $dst, $src\t# int ndd" %}
9980 ins_encode %{
9981 __ eincl($dst$$Register, $src$$Register, false);
9982 %}
9983 ins_pipe(ialu_reg);
9984 %}
9985
9986 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
9987 %{
9988 predicate(UseIncDec);
9989 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
9990 effect(KILL cr);
9991
9992 ins_cost(125); // XXX
9993 format %{ "incl $dst\t# int" %}
9994 ins_encode %{
9995 __ incrementl($dst$$Address);
9996 %}
9997 ins_pipe(ialu_mem_imm);
9998 %}
9999
10000 // XXX why does that use AddI
10001 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
10002 %{
10003 predicate(!UseAPX && UseIncDec);
10004 match(Set dst (AddI dst src));
10005 effect(KILL cr);
10006
10007 format %{ "decl $dst\t# int" %}
10008 ins_encode %{
10009 __ decrementl($dst$$Register);
10010 %}
10011 ins_pipe(ialu_reg);
10012 %}
10013
10014 instruct decI_rReg_ndd(rRegI dst, rRegI src, immI_M1 val, rFlagsReg cr)
10015 %{
10016 predicate(UseAPX && UseIncDec);
10017 match(Set dst (AddI src val));
10018 effect(KILL cr);
10019 flag(PD::Flag_ndd_demotable_opr1);
10020
10021 format %{ "edecl $dst, $src\t# int ndd" %}
10022 ins_encode %{
10023 __ edecl($dst$$Register, $src$$Register, false);
10024 %}
10025 ins_pipe(ialu_reg);
10026 %}
10027
10028 // XXX why does that use AddI
10029 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
10030 %{
10031 predicate(UseIncDec);
10032 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
10033 effect(KILL cr);
10034
10035 ins_cost(125); // XXX
10036 format %{ "decl $dst\t# int" %}
10037 ins_encode %{
10038 __ decrementl($dst$$Address);
10039 %}
10040 ins_pipe(ialu_mem_imm);
10041 %}
10042
10043 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
10044 %{
10045 predicate(VM_Version::supports_fast_2op_lea());
10046 match(Set dst (AddI (LShiftI index scale) disp));
10047
10048 format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
10049 ins_encode %{
10050 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10051 __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
10052 %}
10053 ins_pipe(ialu_reg_reg);
10054 %}
10055
10056 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
10057 %{
10058 predicate(VM_Version::supports_fast_3op_lea());
10059 match(Set dst (AddI (AddI base index) disp));
10060
10061 format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
10062 ins_encode %{
10063 __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
10064 %}
10065 ins_pipe(ialu_reg_reg);
10066 %}
10067
10068 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
10069 %{
10070 predicate(VM_Version::supports_fast_2op_lea());
10071 match(Set dst (AddI base (LShiftI index scale)));
10072
10073 format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
10074 ins_encode %{
10075 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10076 __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
10077 %}
10078 ins_pipe(ialu_reg_reg);
10079 %}
10080
10081 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
10082 %{
10083 predicate(VM_Version::supports_fast_3op_lea());
10084 match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
10085
10086 format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
10087 ins_encode %{
10088 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10089 __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
10090 %}
10091 ins_pipe(ialu_reg_reg);
10092 %}
10093
10094 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10095 %{
10096 predicate(!UseAPX);
10097 match(Set dst (AddL dst src));
10098 effect(KILL cr);
10099 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10100
10101 format %{ "addq $dst, $src\t# long" %}
10102 ins_encode %{
10103 __ addq($dst$$Register, $src$$Register);
10104 %}
10105 ins_pipe(ialu_reg_reg);
10106 %}
10107
10108 instruct addL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
10109 %{
10110 predicate(UseAPX);
10111 match(Set dst (AddL src1 src2));
10112 effect(KILL cr);
10113 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
10114
10115 format %{ "eaddq $dst, $src1, $src2\t# long ndd" %}
10116 ins_encode %{
10117 __ eaddq($dst$$Register, $src1$$Register, $src2$$Register, false);
10118 %}
10119 ins_pipe(ialu_reg_reg);
10120 %}
10121
10122 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10123 %{
10124 predicate(!UseAPX);
10125 match(Set dst (AddL dst src));
10126 effect(KILL cr);
10127 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10128
10129 format %{ "addq $dst, $src\t# long" %}
10130 ins_encode %{
10131 __ addq($dst$$Register, $src$$constant);
10132 %}
10133 ins_pipe( ialu_reg );
10134 %}
10135
10136 instruct addL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
10137 %{
10138 predicate(UseAPX);
10139 match(Set dst (AddL src1 src2));
10140 effect(KILL cr);
10141 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
10142
10143 format %{ "eaddq $dst, $src1, $src2\t# long ndd" %}
10144 ins_encode %{
10145 __ eaddq($dst$$Register, $src1$$Register, $src2$$constant, false);
10146 %}
10147 ins_pipe( ialu_reg );
10148 %}
10149
10150 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10151 %{
10152 match(Set dst (AddL dst (LoadL src)));
10153 effect(KILL cr);
10154 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10155
10156 ins_cost(150); // XXX
10157 format %{ "addq $dst, $src\t# long" %}
10158 ins_encode %{
10159 __ addq($dst$$Register, $src$$Address);
10160 %}
10161 ins_pipe(ialu_reg_mem);
10162 %}
10163
10164 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10165 %{
10166 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10167 effect(KILL cr);
10168 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10169
10170 ins_cost(150); // XXX
10171 format %{ "addq $dst, $src\t# long" %}
10172 ins_encode %{
10173 __ addq($dst$$Address, $src$$Register);
10174 %}
10175 ins_pipe(ialu_mem_reg);
10176 %}
10177
10178 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10179 %{
10180 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10181 effect(KILL cr);
10182 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10183
10184 ins_cost(125); // XXX
10185 format %{ "addq $dst, $src\t# long" %}
10186 ins_encode %{
10187 __ addq($dst$$Address, $src$$constant);
10188 %}
10189 ins_pipe(ialu_mem_imm);
10190 %}
10191
10192 instruct incL_rReg(rRegL dst, immL1 src, rFlagsReg cr)
10193 %{
10194 predicate(!UseAPX && UseIncDec);
10195 match(Set dst (AddL dst src));
10196 effect(KILL cr);
10197
10198 format %{ "incq $dst\t# long" %}
10199 ins_encode %{
10200 __ incrementq($dst$$Register);
10201 %}
10202 ins_pipe(ialu_reg);
10203 %}
10204
10205 instruct incL_rReg_ndd(rRegL dst, rRegI src, immL1 val, rFlagsReg cr)
10206 %{
10207 predicate(UseAPX && UseIncDec);
10208 match(Set dst (AddL src val));
10209 effect(KILL cr);
10210 flag(PD::Flag_ndd_demotable_opr1);
10211
10212 format %{ "eincq $dst, $src\t# long ndd" %}
10213 ins_encode %{
10214 __ eincq($dst$$Register, $src$$Register, false);
10215 %}
10216 ins_pipe(ialu_reg);
10217 %}
10218
10219 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
10220 %{
10221 predicate(UseIncDec);
10222 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10223 effect(KILL cr);
10224
10225 ins_cost(125); // XXX
10226 format %{ "incq $dst\t# long" %}
10227 ins_encode %{
10228 __ incrementq($dst$$Address);
10229 %}
10230 ins_pipe(ialu_mem_imm);
10231 %}
10232
10233 // XXX why does that use AddL
10234 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
10235 %{
10236 predicate(!UseAPX && UseIncDec);
10237 match(Set dst (AddL dst src));
10238 effect(KILL cr);
10239
10240 format %{ "decq $dst\t# long" %}
10241 ins_encode %{
10242 __ decrementq($dst$$Register);
10243 %}
10244 ins_pipe(ialu_reg);
10245 %}
10246
10247 instruct decL_rReg_ndd(rRegL dst, rRegL src, immL_M1 val, rFlagsReg cr)
10248 %{
10249 predicate(UseAPX && UseIncDec);
10250 match(Set dst (AddL src val));
10251 effect(KILL cr);
10252 flag(PD::Flag_ndd_demotable_opr1);
10253
10254 format %{ "edecq $dst, $src\t# long ndd" %}
10255 ins_encode %{
10256 __ edecq($dst$$Register, $src$$Register, false);
10257 %}
10258 ins_pipe(ialu_reg);
10259 %}
10260
10261 // XXX why does that use AddL
10262 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
10263 %{
10264 predicate(UseIncDec);
10265 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10266 effect(KILL cr);
10267
10268 ins_cost(125); // XXX
10269 format %{ "decq $dst\t# long" %}
10270 ins_encode %{
10271 __ decrementq($dst$$Address);
10272 %}
10273 ins_pipe(ialu_mem_imm);
10274 %}
10275
10276 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
10277 %{
10278 predicate(VM_Version::supports_fast_2op_lea());
10279 match(Set dst (AddL (LShiftL index scale) disp));
10280
10281 format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
10282 ins_encode %{
10283 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10284 __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
10285 %}
10286 ins_pipe(ialu_reg_reg);
10287 %}
10288
10289 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
10290 %{
10291 predicate(VM_Version::supports_fast_3op_lea());
10292 match(Set dst (AddL (AddL base index) disp));
10293
10294 format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
10295 ins_encode %{
10296 __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
10297 %}
10298 ins_pipe(ialu_reg_reg);
10299 %}
10300
10301 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
10302 %{
10303 predicate(VM_Version::supports_fast_2op_lea());
10304 match(Set dst (AddL base (LShiftL index scale)));
10305
10306 format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
10307 ins_encode %{
10308 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10309 __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
10310 %}
10311 ins_pipe(ialu_reg_reg);
10312 %}
10313
10314 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
10315 %{
10316 predicate(VM_Version::supports_fast_3op_lea());
10317 match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
10318
10319 format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
10320 ins_encode %{
10321 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10322 __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
10323 %}
10324 ins_pipe(ialu_reg_reg);
10325 %}
10326
10327 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
10328 %{
10329 match(Set dst (AddP dst src));
10330 effect(KILL cr);
10331 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10332
10333 format %{ "addq $dst, $src\t# ptr" %}
10334 ins_encode %{
10335 __ addq($dst$$Register, $src$$Register);
10336 %}
10337 ins_pipe(ialu_reg_reg);
10338 %}
10339
10340 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
10341 %{
10342 match(Set dst (AddP dst src));
10343 effect(KILL cr);
10344 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10345
10346 format %{ "addq $dst, $src\t# ptr" %}
10347 ins_encode %{
10348 __ addq($dst$$Register, $src$$constant);
10349 %}
10350 ins_pipe( ialu_reg );
10351 %}
10352
10353 // XXX addP mem ops ????
10354
10355 instruct checkCastPP(rRegP dst)
10356 %{
10357 match(Set dst (CheckCastPP dst));
10358
10359 size(0);
10360 format %{ "# checkcastPP of $dst" %}
10361 ins_encode(/* empty encoding */);
10362 ins_pipe(empty);
10363 %}
10364
10365 instruct castPP(rRegP dst)
10366 %{
10367 match(Set dst (CastPP dst));
10368
10369 size(0);
10370 format %{ "# castPP of $dst" %}
10371 ins_encode(/* empty encoding */);
10372 ins_pipe(empty);
10373 %}
10374
10375 instruct castII(rRegI dst)
10376 %{
10377 predicate(VerifyConstraintCasts == 0);
10378 match(Set dst (CastII dst));
10379
10380 size(0);
10381 format %{ "# castII of $dst" %}
10382 ins_encode(/* empty encoding */);
10383 ins_cost(0);
10384 ins_pipe(empty);
10385 %}
10386
10387 instruct castII_checked(rRegI dst, rFlagsReg cr)
10388 %{
10389 predicate(VerifyConstraintCasts > 0);
10390 match(Set dst (CastII dst));
10391
10392 effect(KILL cr);
10393 format %{ "# cast_checked_II $dst" %}
10394 ins_encode %{
10395 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register);
10396 %}
10397 ins_pipe(pipe_slow);
10398 %}
10399
10400 instruct castLL(rRegL dst)
10401 %{
10402 predicate(VerifyConstraintCasts == 0);
10403 match(Set dst (CastLL dst));
10404
10405 size(0);
10406 format %{ "# castLL of $dst" %}
10407 ins_encode(/* empty encoding */);
10408 ins_cost(0);
10409 ins_pipe(empty);
10410 %}
10411
10412 instruct castLL_checked_L32(rRegL dst, rFlagsReg cr)
10413 %{
10414 predicate(VerifyConstraintCasts > 0 && castLL_is_imm32(n));
10415 match(Set dst (CastLL dst));
10416
10417 effect(KILL cr);
10418 format %{ "# cast_checked_LL $dst" %}
10419 ins_encode %{
10420 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, noreg);
10421 %}
10422 ins_pipe(pipe_slow);
10423 %}
10424
10425 instruct castLL_checked(rRegL dst, rRegL tmp, rFlagsReg cr)
10426 %{
10427 predicate(VerifyConstraintCasts > 0 && !castLL_is_imm32(n));
10428 match(Set dst (CastLL dst));
10429
10430 effect(KILL cr, TEMP tmp);
10431 format %{ "# cast_checked_LL $dst\tusing $tmp as TEMP" %}
10432 ins_encode %{
10433 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, $tmp$$Register);
10434 %}
10435 ins_pipe(pipe_slow);
10436 %}
10437
10438 instruct castFF(regF dst)
10439 %{
10440 match(Set dst (CastFF dst));
10441
10442 size(0);
10443 format %{ "# castFF of $dst" %}
10444 ins_encode(/* empty encoding */);
10445 ins_cost(0);
10446 ins_pipe(empty);
10447 %}
10448
10449 instruct castHH(regF dst)
10450 %{
10451 match(Set dst (CastHH dst));
10452
10453 size(0);
10454 format %{ "# castHH of $dst" %}
10455 ins_encode(/* empty encoding */);
10456 ins_cost(0);
10457 ins_pipe(empty);
10458 %}
10459
10460 instruct castDD(regD dst)
10461 %{
10462 match(Set dst (CastDD dst));
10463
10464 size(0);
10465 format %{ "# castDD of $dst" %}
10466 ins_encode(/* empty encoding */);
10467 ins_cost(0);
10468 ins_pipe(empty);
10469 %}
10470
10471 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
10472 instruct compareAndSwapP(rRegI res,
10473 memory mem_ptr,
10474 rax_RegP oldval, rRegP newval,
10475 rFlagsReg cr)
10476 %{
10477 predicate(n->as_LoadStore()->barrier_data() == 0);
10478 match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
10479 match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
10480 effect(KILL cr, KILL oldval);
10481
10482 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10483 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10484 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10485 ins_encode %{
10486 __ lock();
10487 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10488 __ setcc(Assembler::equal, $res$$Register);
10489 %}
10490 ins_pipe( pipe_cmpxchg );
10491 %}
10492
10493 instruct compareAndSwapL(rRegI res,
10494 memory mem_ptr,
10495 rax_RegL oldval, rRegL newval,
10496 rFlagsReg cr)
10497 %{
10498 match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
10499 match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
10500 effect(KILL cr, KILL oldval);
10501
10502 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10503 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10504 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10505 ins_encode %{
10506 __ lock();
10507 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10508 __ setcc(Assembler::equal, $res$$Register);
10509 %}
10510 ins_pipe( pipe_cmpxchg );
10511 %}
10512
10513 instruct compareAndSwapI(rRegI res,
10514 memory mem_ptr,
10515 rax_RegI oldval, rRegI newval,
10516 rFlagsReg cr)
10517 %{
10518 match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
10519 match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
10520 effect(KILL cr, KILL oldval);
10521
10522 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10523 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10524 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10525 ins_encode %{
10526 __ lock();
10527 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10528 __ setcc(Assembler::equal, $res$$Register);
10529 %}
10530 ins_pipe( pipe_cmpxchg );
10531 %}
10532
10533 instruct compareAndSwapB(rRegI res,
10534 memory mem_ptr,
10535 rax_RegI oldval, rRegI newval,
10536 rFlagsReg cr)
10537 %{
10538 match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
10539 match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
10540 effect(KILL cr, KILL oldval);
10541
10542 format %{ "cmpxchgb $mem_ptr,$newval\t# "
10543 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10544 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10545 ins_encode %{
10546 __ lock();
10547 __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
10548 __ setcc(Assembler::equal, $res$$Register);
10549 %}
10550 ins_pipe( pipe_cmpxchg );
10551 %}
10552
10553 instruct compareAndSwapS(rRegI res,
10554 memory mem_ptr,
10555 rax_RegI oldval, rRegI newval,
10556 rFlagsReg cr)
10557 %{
10558 match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
10559 match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
10560 effect(KILL cr, KILL oldval);
10561
10562 format %{ "cmpxchgw $mem_ptr,$newval\t# "
10563 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10564 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10565 ins_encode %{
10566 __ lock();
10567 __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
10568 __ setcc(Assembler::equal, $res$$Register);
10569 %}
10570 ins_pipe( pipe_cmpxchg );
10571 %}
10572
10573 instruct compareAndSwapN(rRegI res,
10574 memory mem_ptr,
10575 rax_RegN oldval, rRegN newval,
10576 rFlagsReg cr) %{
10577 predicate(n->as_LoadStore()->barrier_data() == 0);
10578 match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
10579 match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
10580 effect(KILL cr, KILL oldval);
10581
10582 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10583 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10584 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10585 ins_encode %{
10586 __ lock();
10587 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10588 __ setcc(Assembler::equal, $res$$Register);
10589 %}
10590 ins_pipe( pipe_cmpxchg );
10591 %}
10592
10593 instruct compareAndExchangeB(
10594 memory mem_ptr,
10595 rax_RegI oldval, rRegI newval,
10596 rFlagsReg cr)
10597 %{
10598 match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
10599 effect(KILL cr);
10600
10601 format %{ "cmpxchgb $mem_ptr,$newval\t# "
10602 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10603 ins_encode %{
10604 __ lock();
10605 __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
10606 %}
10607 ins_pipe( pipe_cmpxchg );
10608 %}
10609
10610 instruct compareAndExchangeS(
10611 memory mem_ptr,
10612 rax_RegI oldval, rRegI newval,
10613 rFlagsReg cr)
10614 %{
10615 match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
10616 effect(KILL cr);
10617
10618 format %{ "cmpxchgw $mem_ptr,$newval\t# "
10619 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10620 ins_encode %{
10621 __ lock();
10622 __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
10623 %}
10624 ins_pipe( pipe_cmpxchg );
10625 %}
10626
10627 instruct compareAndExchangeI(
10628 memory mem_ptr,
10629 rax_RegI oldval, rRegI newval,
10630 rFlagsReg cr)
10631 %{
10632 match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
10633 effect(KILL cr);
10634
10635 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10636 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10637 ins_encode %{
10638 __ lock();
10639 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10640 %}
10641 ins_pipe( pipe_cmpxchg );
10642 %}
10643
10644 instruct compareAndExchangeL(
10645 memory mem_ptr,
10646 rax_RegL oldval, rRegL newval,
10647 rFlagsReg cr)
10648 %{
10649 match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
10650 effect(KILL cr);
10651
10652 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10653 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10654 ins_encode %{
10655 __ lock();
10656 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10657 %}
10658 ins_pipe( pipe_cmpxchg );
10659 %}
10660
10661 instruct compareAndExchangeN(
10662 memory mem_ptr,
10663 rax_RegN oldval, rRegN newval,
10664 rFlagsReg cr) %{
10665 predicate(n->as_LoadStore()->barrier_data() == 0);
10666 match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
10667 effect(KILL cr);
10668
10669 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10670 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10671 ins_encode %{
10672 __ lock();
10673 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10674 %}
10675 ins_pipe( pipe_cmpxchg );
10676 %}
10677
10678 instruct compareAndExchangeP(
10679 memory mem_ptr,
10680 rax_RegP oldval, rRegP newval,
10681 rFlagsReg cr)
10682 %{
10683 predicate(n->as_LoadStore()->barrier_data() == 0);
10684 match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
10685 effect(KILL cr);
10686
10687 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10688 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10689 ins_encode %{
10690 __ lock();
10691 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10692 %}
10693 ins_pipe( pipe_cmpxchg );
10694 %}
10695
10696 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
10697 predicate(n->as_LoadStore()->result_not_used());
10698 match(Set dummy (GetAndAddB mem add));
10699 effect(KILL cr);
10700 format %{ "addb_lock $mem, $add" %}
10701 ins_encode %{
10702 __ lock();
10703 __ addb($mem$$Address, $add$$Register);
10704 %}
10705 ins_pipe(pipe_cmpxchg);
10706 %}
10707
10708 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
10709 predicate(n->as_LoadStore()->result_not_used());
10710 match(Set dummy (GetAndAddB mem add));
10711 effect(KILL cr);
10712 format %{ "addb_lock $mem, $add" %}
10713 ins_encode %{
10714 __ lock();
10715 __ addb($mem$$Address, $add$$constant);
10716 %}
10717 ins_pipe(pipe_cmpxchg);
10718 %}
10719
10720 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
10721 predicate(!n->as_LoadStore()->result_not_used());
10722 match(Set newval (GetAndAddB mem newval));
10723 effect(KILL cr);
10724 format %{ "xaddb_lock $mem, $newval\t# $newval -> byte" %}
10725 ins_encode %{
10726 __ lock();
10727 __ xaddb($mem$$Address, $newval$$Register);
10728 __ narrow_subword_type($newval$$Register, T_BYTE);
10729 %}
10730 ins_pipe(pipe_cmpxchg);
10731 %}
10732
10733 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
10734 predicate(n->as_LoadStore()->result_not_used());
10735 match(Set dummy (GetAndAddS mem add));
10736 effect(KILL cr);
10737 format %{ "addw_lock $mem, $add" %}
10738 ins_encode %{
10739 __ lock();
10740 __ addw($mem$$Address, $add$$Register);
10741 %}
10742 ins_pipe(pipe_cmpxchg);
10743 %}
10744
10745 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
10746 predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
10747 match(Set dummy (GetAndAddS mem add));
10748 effect(KILL cr);
10749 format %{ "addw_lock $mem, $add" %}
10750 ins_encode %{
10751 __ lock();
10752 __ addw($mem$$Address, $add$$constant);
10753 %}
10754 ins_pipe(pipe_cmpxchg);
10755 %}
10756
10757 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
10758 predicate(!n->as_LoadStore()->result_not_used());
10759 match(Set newval (GetAndAddS mem newval));
10760 effect(KILL cr);
10761 format %{ "xaddw_lock $mem, $newval\t# $newval -> short" %}
10762 ins_encode %{
10763 __ lock();
10764 __ xaddw($mem$$Address, $newval$$Register);
10765 __ narrow_subword_type($newval$$Register, T_SHORT);
10766 %}
10767 ins_pipe(pipe_cmpxchg);
10768 %}
10769
10770 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
10771 predicate(n->as_LoadStore()->result_not_used());
10772 match(Set dummy (GetAndAddI mem add));
10773 effect(KILL cr);
10774 format %{ "addl_lock $mem, $add" %}
10775 ins_encode %{
10776 __ lock();
10777 __ addl($mem$$Address, $add$$Register);
10778 %}
10779 ins_pipe(pipe_cmpxchg);
10780 %}
10781
10782 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
10783 predicate(n->as_LoadStore()->result_not_used());
10784 match(Set dummy (GetAndAddI mem add));
10785 effect(KILL cr);
10786 format %{ "addl_lock $mem, $add" %}
10787 ins_encode %{
10788 __ lock();
10789 __ addl($mem$$Address, $add$$constant);
10790 %}
10791 ins_pipe(pipe_cmpxchg);
10792 %}
10793
10794 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
10795 predicate(!n->as_LoadStore()->result_not_used());
10796 match(Set newval (GetAndAddI mem newval));
10797 effect(KILL cr);
10798 format %{ "xaddl_lock $mem, $newval" %}
10799 ins_encode %{
10800 __ lock();
10801 __ xaddl($mem$$Address, $newval$$Register);
10802 %}
10803 ins_pipe(pipe_cmpxchg);
10804 %}
10805
10806 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
10807 predicate(n->as_LoadStore()->result_not_used());
10808 match(Set dummy (GetAndAddL mem add));
10809 effect(KILL cr);
10810 format %{ "addq_lock $mem, $add" %}
10811 ins_encode %{
10812 __ lock();
10813 __ addq($mem$$Address, $add$$Register);
10814 %}
10815 ins_pipe(pipe_cmpxchg);
10816 %}
10817
10818 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
10819 predicate(n->as_LoadStore()->result_not_used());
10820 match(Set dummy (GetAndAddL mem add));
10821 effect(KILL cr);
10822 format %{ "addq_lock $mem, $add" %}
10823 ins_encode %{
10824 __ lock();
10825 __ addq($mem$$Address, $add$$constant);
10826 %}
10827 ins_pipe(pipe_cmpxchg);
10828 %}
10829
10830 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
10831 predicate(!n->as_LoadStore()->result_not_used());
10832 match(Set newval (GetAndAddL mem newval));
10833 effect(KILL cr);
10834 format %{ "xaddq_lock $mem, $newval" %}
10835 ins_encode %{
10836 __ lock();
10837 __ xaddq($mem$$Address, $newval$$Register);
10838 %}
10839 ins_pipe(pipe_cmpxchg);
10840 %}
10841
10842 instruct xchgB( memory mem, rRegI newval) %{
10843 match(Set newval (GetAndSetB mem newval));
10844 format %{ "XCHGB $newval,[$mem]\t# $newval -> byte" %}
10845 ins_encode %{
10846 __ xchgb($newval$$Register, $mem$$Address);
10847 __ narrow_subword_type($newval$$Register, T_BYTE);
10848 %}
10849 ins_pipe( pipe_cmpxchg );
10850 %}
10851
10852 instruct xchgS( memory mem, rRegI newval) %{
10853 match(Set newval (GetAndSetS mem newval));
10854 format %{ "XCHGW $newval,[$mem]\t# $newval -> short" %}
10855 ins_encode %{
10856 __ xchgw($newval$$Register, $mem$$Address);
10857 __ narrow_subword_type($newval$$Register, T_SHORT);
10858 %}
10859 ins_pipe( pipe_cmpxchg );
10860 %}
10861
10862 instruct xchgI( memory mem, rRegI newval) %{
10863 match(Set newval (GetAndSetI mem newval));
10864 format %{ "XCHGL $newval,[$mem]" %}
10865 ins_encode %{
10866 __ xchgl($newval$$Register, $mem$$Address);
10867 %}
10868 ins_pipe( pipe_cmpxchg );
10869 %}
10870
10871 instruct xchgL( memory mem, rRegL newval) %{
10872 match(Set newval (GetAndSetL mem newval));
10873 format %{ "XCHGL $newval,[$mem]" %}
10874 ins_encode %{
10875 __ xchgq($newval$$Register, $mem$$Address);
10876 %}
10877 ins_pipe( pipe_cmpxchg );
10878 %}
10879
10880 instruct xchgP( memory mem, rRegP newval) %{
10881 match(Set newval (GetAndSetP mem newval));
10882 predicate(n->as_LoadStore()->barrier_data() == 0);
10883 format %{ "XCHGQ $newval,[$mem]" %}
10884 ins_encode %{
10885 __ xchgq($newval$$Register, $mem$$Address);
10886 %}
10887 ins_pipe( pipe_cmpxchg );
10888 %}
10889
10890 instruct xchgN( memory mem, rRegN newval) %{
10891 predicate(n->as_LoadStore()->barrier_data() == 0);
10892 match(Set newval (GetAndSetN mem newval));
10893 format %{ "XCHGL $newval,$mem]" %}
10894 ins_encode %{
10895 __ xchgl($newval$$Register, $mem$$Address);
10896 %}
10897 ins_pipe( pipe_cmpxchg );
10898 %}
10899
10900 //----------Abs Instructions-------------------------------------------
10901
10902 // Integer Absolute Instructions
10903 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10904 %{
10905 match(Set dst (AbsI src));
10906 effect(TEMP dst, KILL cr);
10907 format %{ "xorl $dst, $dst\t# abs int\n\t"
10908 "subl $dst, $src\n\t"
10909 "cmovll $dst, $src" %}
10910 ins_encode %{
10911 __ xorl($dst$$Register, $dst$$Register);
10912 __ subl($dst$$Register, $src$$Register);
10913 __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
10914 %}
10915
10916 ins_pipe(ialu_reg_reg);
10917 %}
10918
10919 // Long Absolute Instructions
10920 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10921 %{
10922 match(Set dst (AbsL src));
10923 effect(TEMP dst, KILL cr);
10924 format %{ "xorl $dst, $dst\t# abs long\n\t"
10925 "subq $dst, $src\n\t"
10926 "cmovlq $dst, $src" %}
10927 ins_encode %{
10928 __ xorl($dst$$Register, $dst$$Register);
10929 __ subq($dst$$Register, $src$$Register);
10930 __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
10931 %}
10932
10933 ins_pipe(ialu_reg_reg);
10934 %}
10935
10936 //----------Subtraction Instructions-------------------------------------------
10937
10938 // Integer Subtraction Instructions
10939 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10940 %{
10941 predicate(!UseAPX);
10942 match(Set dst (SubI dst src));
10943 effect(KILL cr);
10944 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10945
10946 format %{ "subl $dst, $src\t# int" %}
10947 ins_encode %{
10948 __ subl($dst$$Register, $src$$Register);
10949 %}
10950 ins_pipe(ialu_reg_reg);
10951 %}
10952
10953 instruct subI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10954 %{
10955 predicate(UseAPX);
10956 match(Set dst (SubI src1 src2));
10957 effect(KILL cr);
10958 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
10959
10960 format %{ "esubl $dst, $src1, $src2\t# int ndd" %}
10961 ins_encode %{
10962 __ esubl($dst$$Register, $src1$$Register, $src2$$Register, false);
10963 %}
10964 ins_pipe(ialu_reg_reg);
10965 %}
10966
10967 instruct subI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10968 %{
10969 predicate(UseAPX);
10970 match(Set dst (SubI src1 src2));
10971 effect(KILL cr);
10972 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
10973
10974 format %{ "esubl $dst, $src1, $src2\t# int ndd" %}
10975 ins_encode %{
10976 __ esubl($dst$$Register, $src1$$Register, $src2$$constant, false);
10977 %}
10978 ins_pipe(ialu_reg_reg);
10979 %}
10980
10981 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10982 %{
10983 match(Set dst (SubI dst (LoadI src)));
10984 effect(KILL cr);
10985 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10986
10987 ins_cost(150);
10988 format %{ "subl $dst, $src\t# int" %}
10989 ins_encode %{
10990 __ subl($dst$$Register, $src$$Address);
10991 %}
10992 ins_pipe(ialu_reg_mem);
10993 %}
10994
10995 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10996 %{
10997 match(Set dst (StoreI dst (SubI (LoadI dst) src)));
10998 effect(KILL cr);
10999 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11000
11001 ins_cost(150);
11002 format %{ "subl $dst, $src\t# int" %}
11003 ins_encode %{
11004 __ subl($dst$$Address, $src$$Register);
11005 %}
11006 ins_pipe(ialu_mem_reg);
11007 %}
11008
11009 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11010 %{
11011 predicate(!UseAPX);
11012 match(Set dst (SubL dst src));
11013 effect(KILL cr);
11014 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11015
11016 format %{ "subq $dst, $src\t# long" %}
11017 ins_encode %{
11018 __ subq($dst$$Register, $src$$Register);
11019 %}
11020 ins_pipe(ialu_reg_reg);
11021 %}
11022
11023 instruct subL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11024 %{
11025 predicate(UseAPX);
11026 match(Set dst (SubL src1 src2));
11027 effect(KILL cr);
11028 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11029
11030 format %{ "esubq $dst, $src1, $src2\t# long ndd" %}
11031 ins_encode %{
11032 __ esubq($dst$$Register, $src1$$Register, $src2$$Register, false);
11033 %}
11034 ins_pipe(ialu_reg_reg);
11035 %}
11036
11037 instruct subL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11038 %{
11039 predicate(UseAPX);
11040 match(Set dst (SubL src1 src2));
11041 effect(KILL cr);
11042 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11043
11044 format %{ "esubq $dst, $src1, $src2\t# long ndd" %}
11045 ins_encode %{
11046 __ esubq($dst$$Register, $src1$$Register, $src2$$constant, false);
11047 %}
11048 ins_pipe(ialu_reg_reg);
11049 %}
11050
11051 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11052 %{
11053 match(Set dst (SubL dst (LoadL src)));
11054 effect(KILL cr);
11055 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11056
11057 ins_cost(150);
11058 format %{ "subq $dst, $src\t# long" %}
11059 ins_encode %{
11060 __ subq($dst$$Register, $src$$Address);
11061 %}
11062 ins_pipe(ialu_reg_mem);
11063 %}
11064
11065 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11066 %{
11067 match(Set dst (StoreL dst (SubL (LoadL dst) src)));
11068 effect(KILL cr);
11069 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11070
11071 ins_cost(150);
11072 format %{ "subq $dst, $src\t# long" %}
11073 ins_encode %{
11074 __ subq($dst$$Address, $src$$Register);
11075 %}
11076 ins_pipe(ialu_mem_reg);
11077 %}
11078
11079 // Subtract from a pointer
11080 // XXX hmpf???
11081 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
11082 %{
11083 match(Set dst (AddP dst (SubI zero src)));
11084 effect(KILL cr);
11085
11086 format %{ "subq $dst, $src\t# ptr - int" %}
11087 ins_encode %{
11088 __ subq($dst$$Register, $src$$Register);
11089 %}
11090 ins_pipe(ialu_reg_reg);
11091 %}
11092
11093 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
11094 %{
11095 predicate(!UseAPX);
11096 match(Set dst (SubI zero dst));
11097 effect(KILL cr);
11098 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11099
11100 format %{ "negl $dst\t# int" %}
11101 ins_encode %{
11102 __ negl($dst$$Register);
11103 %}
11104 ins_pipe(ialu_reg);
11105 %}
11106
11107 instruct negI_rReg_ndd(rRegI dst, rRegI src, immI_0 zero, rFlagsReg cr)
11108 %{
11109 predicate(UseAPX);
11110 match(Set dst (SubI zero src));
11111 effect(KILL cr);
11112 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr2);
11113
11114 format %{ "enegl $dst, $src\t# int ndd" %}
11115 ins_encode %{
11116 __ enegl($dst$$Register, $src$$Register, false);
11117 %}
11118 ins_pipe(ialu_reg);
11119 %}
11120
11121 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
11122 %{
11123 predicate(!UseAPX);
11124 match(Set dst (NegI dst));
11125 effect(KILL cr);
11126 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11127
11128 format %{ "negl $dst\t# int" %}
11129 ins_encode %{
11130 __ negl($dst$$Register);
11131 %}
11132 ins_pipe(ialu_reg);
11133 %}
11134
11135 instruct negI_rReg_2_ndd(rRegI dst, rRegI src, rFlagsReg cr)
11136 %{
11137 predicate(UseAPX);
11138 match(Set dst (NegI src));
11139 effect(KILL cr);
11140 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11141
11142 format %{ "enegl $dst, $src\t# int ndd" %}
11143 ins_encode %{
11144 __ enegl($dst$$Register, $src$$Register, false);
11145 %}
11146 ins_pipe(ialu_reg);
11147 %}
11148
11149 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
11150 %{
11151 match(Set dst (StoreI dst (SubI zero (LoadI dst))));
11152 effect(KILL cr);
11153 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11154
11155 format %{ "negl $dst\t# int" %}
11156 ins_encode %{
11157 __ negl($dst$$Address);
11158 %}
11159 ins_pipe(ialu_reg);
11160 %}
11161
11162 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
11163 %{
11164 predicate(!UseAPX);
11165 match(Set dst (SubL zero dst));
11166 effect(KILL cr);
11167 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11168
11169 format %{ "negq $dst\t# long" %}
11170 ins_encode %{
11171 __ negq($dst$$Register);
11172 %}
11173 ins_pipe(ialu_reg);
11174 %}
11175
11176 instruct negL_rReg_ndd(rRegL dst, rRegL src, immL0 zero, rFlagsReg cr)
11177 %{
11178 predicate(UseAPX);
11179 match(Set dst (SubL zero src));
11180 effect(KILL cr);
11181 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr2);
11182
11183 format %{ "enegq $dst, $src\t# long ndd" %}
11184 ins_encode %{
11185 __ enegq($dst$$Register, $src$$Register, false);
11186 %}
11187 ins_pipe(ialu_reg);
11188 %}
11189
11190 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
11191 %{
11192 predicate(!UseAPX);
11193 match(Set dst (NegL dst));
11194 effect(KILL cr);
11195 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11196
11197 format %{ "negq $dst\t# int" %}
11198 ins_encode %{
11199 __ negq($dst$$Register);
11200 %}
11201 ins_pipe(ialu_reg);
11202 %}
11203
11204 instruct negL_rReg_2_ndd(rRegL dst, rRegL src, rFlagsReg cr)
11205 %{
11206 predicate(UseAPX);
11207 match(Set dst (NegL src));
11208 effect(KILL cr);
11209 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11210
11211 format %{ "enegq $dst, $src\t# long ndd" %}
11212 ins_encode %{
11213 __ enegq($dst$$Register, $src$$Register, false);
11214 %}
11215 ins_pipe(ialu_reg);
11216 %}
11217
11218 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
11219 %{
11220 match(Set dst (StoreL dst (SubL zero (LoadL dst))));
11221 effect(KILL cr);
11222 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11223
11224 format %{ "negq $dst\t# long" %}
11225 ins_encode %{
11226 __ negq($dst$$Address);
11227 %}
11228 ins_pipe(ialu_reg);
11229 %}
11230
11231 //----------Multiplication/Division Instructions-------------------------------
11232 // Integer Multiplication Instructions
11233 // Multiply Register
11234
11235 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
11236 %{
11237 predicate(!UseAPX);
11238 match(Set dst (MulI dst src));
11239 effect(KILL cr);
11240
11241 ins_cost(300);
11242 format %{ "imull $dst, $src\t# int" %}
11243 ins_encode %{
11244 __ imull($dst$$Register, $src$$Register);
11245 %}
11246 ins_pipe(ialu_reg_reg_alu0);
11247 %}
11248
11249 instruct mulI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
11250 %{
11251 predicate(UseAPX);
11252 match(Set dst (MulI src1 src2));
11253 effect(KILL cr);
11254 flag(PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
11255
11256 ins_cost(300);
11257 format %{ "eimull $dst, $src1, $src2\t# int ndd" %}
11258 ins_encode %{
11259 __ eimull($dst$$Register, $src1$$Register, $src2$$Register, false);
11260 %}
11261 ins_pipe(ialu_reg_reg_alu0);
11262 %}
11263
11264 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
11265 %{
11266 match(Set dst (MulI src imm));
11267 effect(KILL cr);
11268
11269 ins_cost(300);
11270 format %{ "imull $dst, $src, $imm\t# int" %}
11271 ins_encode %{
11272 __ imull($dst$$Register, $src$$Register, $imm$$constant);
11273 %}
11274 ins_pipe(ialu_reg_reg_alu0);
11275 %}
11276
11277 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
11278 %{
11279 match(Set dst (MulI dst (LoadI src)));
11280 effect(KILL cr);
11281
11282 ins_cost(350);
11283 format %{ "imull $dst, $src\t# int" %}
11284 ins_encode %{
11285 __ imull($dst$$Register, $src$$Address);
11286 %}
11287 ins_pipe(ialu_reg_mem_alu0);
11288 %}
11289
11290 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
11291 %{
11292 match(Set dst (MulI (LoadI src) imm));
11293 effect(KILL cr);
11294
11295 ins_cost(300);
11296 format %{ "imull $dst, $src, $imm\t# int" %}
11297 ins_encode %{
11298 __ imull($dst$$Register, $src$$Address, $imm$$constant);
11299 %}
11300 ins_pipe(ialu_reg_mem_alu0);
11301 %}
11302
11303 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
11304 %{
11305 match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
11306 effect(KILL cr, KILL src2);
11307
11308 expand %{ mulI_rReg(dst, src1, cr);
11309 mulI_rReg(src2, src3, cr);
11310 addI_rReg(dst, src2, cr); %}
11311 %}
11312
11313 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11314 %{
11315 predicate(!UseAPX);
11316 match(Set dst (MulL dst src));
11317 effect(KILL cr);
11318
11319 ins_cost(300);
11320 format %{ "imulq $dst, $src\t# long" %}
11321 ins_encode %{
11322 __ imulq($dst$$Register, $src$$Register);
11323 %}
11324 ins_pipe(ialu_reg_reg_alu0);
11325 %}
11326
11327 instruct mulL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11328 %{
11329 predicate(UseAPX);
11330 match(Set dst (MulL src1 src2));
11331 effect(KILL cr);
11332 flag(PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
11333
11334 ins_cost(300);
11335 format %{ "eimulq $dst, $src1, $src2\t# long ndd" %}
11336 ins_encode %{
11337 __ eimulq($dst$$Register, $src1$$Register, $src2$$Register, false);
11338 %}
11339 ins_pipe(ialu_reg_reg_alu0);
11340 %}
11341
11342 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
11343 %{
11344 match(Set dst (MulL src imm));
11345 effect(KILL cr);
11346
11347 ins_cost(300);
11348 format %{ "imulq $dst, $src, $imm\t# long" %}
11349 ins_encode %{
11350 __ imulq($dst$$Register, $src$$Register, $imm$$constant);
11351 %}
11352 ins_pipe(ialu_reg_reg_alu0);
11353 %}
11354
11355 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
11356 %{
11357 match(Set dst (MulL dst (LoadL src)));
11358 effect(KILL cr);
11359
11360 ins_cost(350);
11361 format %{ "imulq $dst, $src\t# long" %}
11362 ins_encode %{
11363 __ imulq($dst$$Register, $src$$Address);
11364 %}
11365 ins_pipe(ialu_reg_mem_alu0);
11366 %}
11367
11368
11369 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
11370 %{
11371 match(Set dst (MulL (LoadL src) imm));
11372 effect(KILL cr);
11373
11374 ins_cost(300);
11375 format %{ "imulq $dst, $src, $imm\t# long" %}
11376 ins_encode %{
11377 __ imulq($dst$$Register, $src$$Address, $imm$$constant);
11378 %}
11379 ins_pipe(ialu_reg_mem_alu0);
11380 %}
11381
11382 instruct mulHiLoL_rReg(rax_RegL rax, rdx_RegL rdx, rRegL src, rFlagsReg cr)
11383 %{
11384 match(MulHiLoL src rax);
11385 match(MulHiLoL rax src);
11386 effect(KILL cr);
11387
11388 ins_cost(300);
11389 format %{ "imulq RDX:RAX, RAX, $src\t# mulhilo" %}
11390 ins_encode %{
11391 __ imulq($src$$Register);
11392 %}
11393 ins_pipe(ialu_reg_reg_alu0);
11394 %}
11395
11396 instruct umulHiLoL_rReg(rax_RegL rax, rdx_RegL rdx, rRegL src, rFlagsReg cr)
11397 %{
11398 match(UMulHiLoL src rax);
11399 match(UMulHiLoL rax src);
11400 effect(KILL cr);
11401
11402 ins_cost(300);
11403 format %{ "mulq RDX:RAX, RAX, $src\t# umulhilo" %}
11404 ins_encode %{
11405 __ mulq($src$$Register);
11406 %}
11407 ins_pipe(ialu_reg_reg_alu0);
11408 %}
11409
11410 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
11411 %{
11412 match(Set dst (MulHiL src rax));
11413 effect(USE_KILL rax, KILL cr);
11414
11415 ins_cost(300);
11416 format %{ "imulq RDX:RAX, RAX, $src\t# mulhi" %}
11417 ins_encode %{
11418 __ imulq($src$$Register);
11419 %}
11420 ins_pipe(ialu_reg_reg_alu0);
11421 %}
11422
11423 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
11424 %{
11425 match(Set dst (UMulHiL src rax));
11426 effect(USE_KILL rax, KILL cr);
11427
11428 ins_cost(300);
11429 format %{ "mulq RDX:RAX, RAX, $src\t# umulhi" %}
11430 ins_encode %{
11431 __ mulq($src$$Register);
11432 %}
11433 ins_pipe(ialu_reg_reg_alu0);
11434 %}
11435
11436 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
11437 rFlagsReg cr)
11438 %{
11439 match(Set rax (DivI rax div));
11440 effect(KILL rdx, KILL cr);
11441
11442 ins_cost(30*100+10*100); // XXX
11443 format %{ "cmpl rax, 0x80000000\t# idiv\n\t"
11444 "jne,s normal\n\t"
11445 "xorl rdx, rdx\n\t"
11446 "cmpl $div, -1\n\t"
11447 "je,s done\n"
11448 "normal: cdql\n\t"
11449 "idivl $div\n"
11450 "done:" %}
11451 ins_encode(cdql_enc(div));
11452 ins_pipe(ialu_reg_reg_alu0);
11453 %}
11454
11455 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
11456 rFlagsReg cr)
11457 %{
11458 match(Set rax (DivL rax div));
11459 effect(KILL rdx, KILL cr);
11460
11461 ins_cost(30*100+10*100); // XXX
11462 format %{ "movq rdx, 0x8000000000000000\t# ldiv\n\t"
11463 "cmpq rax, rdx\n\t"
11464 "jne,s normal\n\t"
11465 "xorl rdx, rdx\n\t"
11466 "cmpq $div, -1\n\t"
11467 "je,s done\n"
11468 "normal: cdqq\n\t"
11469 "idivq $div\n"
11470 "done:" %}
11471 ins_encode(cdqq_enc(div));
11472 ins_pipe(ialu_reg_reg_alu0);
11473 %}
11474
11475 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
11476 %{
11477 match(Set rax (UDivI rax div));
11478 effect(KILL rdx, KILL cr);
11479
11480 ins_cost(300);
11481 format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
11482 ins_encode %{
11483 __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
11484 %}
11485 ins_pipe(ialu_reg_reg_alu0);
11486 %}
11487
11488 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
11489 %{
11490 match(Set rax (UDivL rax div));
11491 effect(KILL rdx, KILL cr);
11492
11493 ins_cost(300);
11494 format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
11495 ins_encode %{
11496 __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
11497 %}
11498 ins_pipe(ialu_reg_reg_alu0);
11499 %}
11500
11501 // Integer DIVMOD with Register, both quotient and mod results
11502 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
11503 rFlagsReg cr)
11504 %{
11505 match(DivModI rax div);
11506 effect(KILL cr);
11507
11508 ins_cost(30*100+10*100); // XXX
11509 format %{ "cmpl rax, 0x80000000\t# idiv\n\t"
11510 "jne,s normal\n\t"
11511 "xorl rdx, rdx\n\t"
11512 "cmpl $div, -1\n\t"
11513 "je,s done\n"
11514 "normal: cdql\n\t"
11515 "idivl $div\n"
11516 "done:" %}
11517 ins_encode(cdql_enc(div));
11518 ins_pipe(pipe_slow);
11519 %}
11520
11521 // Long DIVMOD with Register, both quotient and mod results
11522 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
11523 rFlagsReg cr)
11524 %{
11525 match(DivModL rax div);
11526 effect(KILL cr);
11527
11528 ins_cost(30*100+10*100); // XXX
11529 format %{ "movq rdx, 0x8000000000000000\t# ldiv\n\t"
11530 "cmpq rax, rdx\n\t"
11531 "jne,s normal\n\t"
11532 "xorl rdx, rdx\n\t"
11533 "cmpq $div, -1\n\t"
11534 "je,s done\n"
11535 "normal: cdqq\n\t"
11536 "idivq $div\n"
11537 "done:" %}
11538 ins_encode(cdqq_enc(div));
11539 ins_pipe(pipe_slow);
11540 %}
11541
11542 // Unsigned integer DIVMOD with Register, both quotient and mod results
11543 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
11544 no_rax_rdx_RegI div, rFlagsReg cr)
11545 %{
11546 match(UDivModI rax div);
11547 effect(TEMP tmp, KILL cr);
11548
11549 ins_cost(300);
11550 format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
11551 "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
11552 %}
11553 ins_encode %{
11554 __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
11555 %}
11556 ins_pipe(pipe_slow);
11557 %}
11558
11559 // Unsigned long DIVMOD with Register, both quotient and mod results
11560 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
11561 no_rax_rdx_RegL div, rFlagsReg cr)
11562 %{
11563 match(UDivModL rax div);
11564 effect(TEMP tmp, KILL cr);
11565
11566 ins_cost(300);
11567 format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
11568 "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
11569 %}
11570 ins_encode %{
11571 __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
11572 %}
11573 ins_pipe(pipe_slow);
11574 %}
11575
11576 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
11577 rFlagsReg cr)
11578 %{
11579 match(Set rdx (ModI rax div));
11580 effect(KILL rax, KILL cr);
11581
11582 ins_cost(300); // XXX
11583 format %{ "cmpl rax, 0x80000000\t# irem\n\t"
11584 "jne,s normal\n\t"
11585 "xorl rdx, rdx\n\t"
11586 "cmpl $div, -1\n\t"
11587 "je,s done\n"
11588 "normal: cdql\n\t"
11589 "idivl $div\n"
11590 "done:" %}
11591 ins_encode(cdql_enc(div));
11592 ins_pipe(ialu_reg_reg_alu0);
11593 %}
11594
11595 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
11596 rFlagsReg cr)
11597 %{
11598 match(Set rdx (ModL rax div));
11599 effect(KILL rax, KILL cr);
11600
11601 ins_cost(300); // XXX
11602 format %{ "movq rdx, 0x8000000000000000\t# lrem\n\t"
11603 "cmpq rax, rdx\n\t"
11604 "jne,s normal\n\t"
11605 "xorl rdx, rdx\n\t"
11606 "cmpq $div, -1\n\t"
11607 "je,s done\n"
11608 "normal: cdqq\n\t"
11609 "idivq $div\n"
11610 "done:" %}
11611 ins_encode(cdqq_enc(div));
11612 ins_pipe(ialu_reg_reg_alu0);
11613 %}
11614
11615 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
11616 %{
11617 match(Set rdx (UModI rax div));
11618 effect(KILL rax, KILL cr);
11619
11620 ins_cost(300);
11621 format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
11622 ins_encode %{
11623 __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
11624 %}
11625 ins_pipe(ialu_reg_reg_alu0);
11626 %}
11627
11628 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
11629 %{
11630 match(Set rdx (UModL rax div));
11631 effect(KILL rax, KILL cr);
11632
11633 ins_cost(300);
11634 format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
11635 ins_encode %{
11636 __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
11637 %}
11638 ins_pipe(ialu_reg_reg_alu0);
11639 %}
11640
11641 // Integer Shift Instructions
11642 // Shift Left by one, two, three
11643 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
11644 %{
11645 predicate(!UseAPX);
11646 match(Set dst (LShiftI dst shift));
11647 effect(KILL cr);
11648
11649 format %{ "sall $dst, $shift" %}
11650 ins_encode %{
11651 __ sall($dst$$Register, $shift$$constant);
11652 %}
11653 ins_pipe(ialu_reg);
11654 %}
11655
11656 // Shift Left by one, two, three
11657 instruct salI_rReg_immI2_ndd(rRegI dst, rRegI src, immI2 shift, rFlagsReg cr)
11658 %{
11659 predicate(UseAPX);
11660 match(Set dst (LShiftI src shift));
11661 effect(KILL cr);
11662 flag(PD::Flag_ndd_demotable_opr1);
11663
11664 format %{ "esall $dst, $src, $shift\t# int(ndd)" %}
11665 ins_encode %{
11666 __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
11667 %}
11668 ins_pipe(ialu_reg);
11669 %}
11670
11671 // Shift Left by 8-bit immediate
11672 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
11673 %{
11674 predicate(!UseAPX);
11675 match(Set dst (LShiftI dst shift));
11676 effect(KILL cr);
11677
11678 format %{ "sall $dst, $shift" %}
11679 ins_encode %{
11680 __ sall($dst$$Register, $shift$$constant);
11681 %}
11682 ins_pipe(ialu_reg);
11683 %}
11684
11685 // Shift Left by 8-bit immediate
11686 instruct salI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
11687 %{
11688 predicate(UseAPX);
11689 match(Set dst (LShiftI src shift));
11690 effect(KILL cr);
11691 flag(PD::Flag_ndd_demotable_opr1);
11692
11693 format %{ "esall $dst, $src, $shift\t# int (ndd)" %}
11694 ins_encode %{
11695 __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
11696 %}
11697 ins_pipe(ialu_reg);
11698 %}
11699
11700 // Shift Left by 8-bit immediate
11701 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
11702 %{
11703 match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
11704 effect(KILL cr);
11705
11706 format %{ "sall $dst, $shift" %}
11707 ins_encode %{
11708 __ sall($dst$$Address, $shift$$constant);
11709 %}
11710 ins_pipe(ialu_mem_imm);
11711 %}
11712
11713 // Shift Left by variable
11714 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
11715 %{
11716 predicate(!VM_Version::supports_bmi2());
11717 match(Set dst (LShiftI dst shift));
11718 effect(KILL cr);
11719
11720 format %{ "sall $dst, $shift" %}
11721 ins_encode %{
11722 __ sall($dst$$Register);
11723 %}
11724 ins_pipe(ialu_reg_reg);
11725 %}
11726
11727 // Shift Left by variable
11728 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
11729 %{
11730 predicate(!VM_Version::supports_bmi2());
11731 match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
11732 effect(KILL cr);
11733
11734 format %{ "sall $dst, $shift" %}
11735 ins_encode %{
11736 __ sall($dst$$Address);
11737 %}
11738 ins_pipe(ialu_mem_reg);
11739 %}
11740
11741 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
11742 %{
11743 predicate(VM_Version::supports_bmi2());
11744 match(Set dst (LShiftI src shift));
11745
11746 format %{ "shlxl $dst, $src, $shift" %}
11747 ins_encode %{
11748 __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
11749 %}
11750 ins_pipe(ialu_reg_reg);
11751 %}
11752
11753 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
11754 %{
11755 predicate(VM_Version::supports_bmi2());
11756 match(Set dst (LShiftI (LoadI src) shift));
11757 ins_cost(175);
11758 format %{ "shlxl $dst, $src, $shift" %}
11759 ins_encode %{
11760 __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
11761 %}
11762 ins_pipe(ialu_reg_mem);
11763 %}
11764
11765 // Arithmetic Shift Right by 8-bit immediate
11766 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
11767 %{
11768 predicate(!UseAPX);
11769 match(Set dst (RShiftI dst shift));
11770 effect(KILL cr);
11771
11772 format %{ "sarl $dst, $shift" %}
11773 ins_encode %{
11774 __ sarl($dst$$Register, $shift$$constant);
11775 %}
11776 ins_pipe(ialu_mem_imm);
11777 %}
11778
11779 // Arithmetic Shift Right by 8-bit immediate
11780 instruct sarI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
11781 %{
11782 predicate(UseAPX);
11783 match(Set dst (RShiftI src shift));
11784 effect(KILL cr);
11785 flag(PD::Flag_ndd_demotable_opr1);
11786
11787 format %{ "esarl $dst, $src, $shift\t# int (ndd)" %}
11788 ins_encode %{
11789 __ esarl($dst$$Register, $src$$Register, $shift$$constant, false);
11790 %}
11791 ins_pipe(ialu_mem_imm);
11792 %}
11793
11794 // Arithmetic Shift Right by 8-bit immediate
11795 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
11796 %{
11797 match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
11798 effect(KILL cr);
11799
11800 format %{ "sarl $dst, $shift" %}
11801 ins_encode %{
11802 __ sarl($dst$$Address, $shift$$constant);
11803 %}
11804 ins_pipe(ialu_mem_imm);
11805 %}
11806
11807 // Arithmetic Shift Right by variable
11808 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
11809 %{
11810 predicate(!VM_Version::supports_bmi2());
11811 match(Set dst (RShiftI dst shift));
11812 effect(KILL cr);
11813
11814 format %{ "sarl $dst, $shift" %}
11815 ins_encode %{
11816 __ sarl($dst$$Register);
11817 %}
11818 ins_pipe(ialu_reg_reg);
11819 %}
11820
11821 // Arithmetic Shift Right by variable
11822 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
11823 %{
11824 predicate(!VM_Version::supports_bmi2());
11825 match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
11826 effect(KILL cr);
11827
11828 format %{ "sarl $dst, $shift" %}
11829 ins_encode %{
11830 __ sarl($dst$$Address);
11831 %}
11832 ins_pipe(ialu_mem_reg);
11833 %}
11834
11835 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
11836 %{
11837 predicate(VM_Version::supports_bmi2());
11838 match(Set dst (RShiftI src shift));
11839
11840 format %{ "sarxl $dst, $src, $shift" %}
11841 ins_encode %{
11842 __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
11843 %}
11844 ins_pipe(ialu_reg_reg);
11845 %}
11846
11847 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
11848 %{
11849 predicate(VM_Version::supports_bmi2());
11850 match(Set dst (RShiftI (LoadI src) shift));
11851 ins_cost(175);
11852 format %{ "sarxl $dst, $src, $shift" %}
11853 ins_encode %{
11854 __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
11855 %}
11856 ins_pipe(ialu_reg_mem);
11857 %}
11858
11859 // Logical Shift Right by 8-bit immediate
11860 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
11861 %{
11862 predicate(!UseAPX);
11863 match(Set dst (URShiftI dst shift));
11864 effect(KILL cr);
11865
11866 format %{ "shrl $dst, $shift" %}
11867 ins_encode %{
11868 __ shrl($dst$$Register, $shift$$constant);
11869 %}
11870 ins_pipe(ialu_reg);
11871 %}
11872
11873 // Logical Shift Right by 8-bit immediate
11874 instruct shrI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
11875 %{
11876 predicate(UseAPX);
11877 match(Set dst (URShiftI src shift));
11878 effect(KILL cr);
11879 flag(PD::Flag_ndd_demotable_opr1);
11880
11881 format %{ "eshrl $dst, $src, $shift\t # int (ndd)" %}
11882 ins_encode %{
11883 __ eshrl($dst$$Register, $src$$Register, $shift$$constant, false);
11884 %}
11885 ins_pipe(ialu_reg);
11886 %}
11887
11888 // Logical Shift Right by 8-bit immediate
11889 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
11890 %{
11891 match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
11892 effect(KILL cr);
11893
11894 format %{ "shrl $dst, $shift" %}
11895 ins_encode %{
11896 __ shrl($dst$$Address, $shift$$constant);
11897 %}
11898 ins_pipe(ialu_mem_imm);
11899 %}
11900
11901 // Logical Shift Right by variable
11902 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
11903 %{
11904 predicate(!VM_Version::supports_bmi2());
11905 match(Set dst (URShiftI dst shift));
11906 effect(KILL cr);
11907
11908 format %{ "shrl $dst, $shift" %}
11909 ins_encode %{
11910 __ shrl($dst$$Register);
11911 %}
11912 ins_pipe(ialu_reg_reg);
11913 %}
11914
11915 // Logical Shift Right by variable
11916 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
11917 %{
11918 predicate(!VM_Version::supports_bmi2());
11919 match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
11920 effect(KILL cr);
11921
11922 format %{ "shrl $dst, $shift" %}
11923 ins_encode %{
11924 __ shrl($dst$$Address);
11925 %}
11926 ins_pipe(ialu_mem_reg);
11927 %}
11928
11929 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
11930 %{
11931 predicate(VM_Version::supports_bmi2());
11932 match(Set dst (URShiftI src shift));
11933
11934 format %{ "shrxl $dst, $src, $shift" %}
11935 ins_encode %{
11936 __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
11937 %}
11938 ins_pipe(ialu_reg_reg);
11939 %}
11940
11941 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
11942 %{
11943 predicate(VM_Version::supports_bmi2());
11944 match(Set dst (URShiftI (LoadI src) shift));
11945 ins_cost(175);
11946 format %{ "shrxl $dst, $src, $shift" %}
11947 ins_encode %{
11948 __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
11949 %}
11950 ins_pipe(ialu_reg_mem);
11951 %}
11952
11953 // Long Shift Instructions
11954 // Shift Left by one, two, three
11955 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
11956 %{
11957 predicate(!UseAPX);
11958 match(Set dst (LShiftL dst shift));
11959 effect(KILL cr);
11960
11961 format %{ "salq $dst, $shift" %}
11962 ins_encode %{
11963 __ salq($dst$$Register, $shift$$constant);
11964 %}
11965 ins_pipe(ialu_reg);
11966 %}
11967
11968 // Shift Left by one, two, three
11969 instruct salL_rReg_immI2_ndd(rRegL dst, rRegL src, immI2 shift, rFlagsReg cr)
11970 %{
11971 predicate(UseAPX);
11972 match(Set dst (LShiftL src shift));
11973 effect(KILL cr);
11974 flag(PD::Flag_ndd_demotable_opr1);
11975
11976 format %{ "esalq $dst, $src, $shift\t# long (ndd)" %}
11977 ins_encode %{
11978 __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
11979 %}
11980 ins_pipe(ialu_reg);
11981 %}
11982
11983 // Shift Left by 8-bit immediate
11984 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
11985 %{
11986 predicate(!UseAPX);
11987 match(Set dst (LShiftL dst shift));
11988 effect(KILL cr);
11989
11990 format %{ "salq $dst, $shift" %}
11991 ins_encode %{
11992 __ salq($dst$$Register, $shift$$constant);
11993 %}
11994 ins_pipe(ialu_reg);
11995 %}
11996
11997 // Shift Left by 8-bit immediate
11998 instruct salL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
11999 %{
12000 predicate(UseAPX);
12001 match(Set dst (LShiftL src shift));
12002 effect(KILL cr);
12003 flag(PD::Flag_ndd_demotable_opr1);
12004
12005 format %{ "esalq $dst, $src, $shift\t# long (ndd)" %}
12006 ins_encode %{
12007 __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
12008 %}
12009 ins_pipe(ialu_reg);
12010 %}
12011
12012 // Shift Left by 8-bit immediate
12013 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
12014 %{
12015 match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
12016 effect(KILL cr);
12017
12018 format %{ "salq $dst, $shift" %}
12019 ins_encode %{
12020 __ salq($dst$$Address, $shift$$constant);
12021 %}
12022 ins_pipe(ialu_mem_imm);
12023 %}
12024
12025 // Shift Left by variable
12026 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12027 %{
12028 predicate(!VM_Version::supports_bmi2());
12029 match(Set dst (LShiftL dst shift));
12030 effect(KILL cr);
12031
12032 format %{ "salq $dst, $shift" %}
12033 ins_encode %{
12034 __ salq($dst$$Register);
12035 %}
12036 ins_pipe(ialu_reg_reg);
12037 %}
12038
12039 // Shift Left by variable
12040 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
12041 %{
12042 predicate(!VM_Version::supports_bmi2());
12043 match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
12044 effect(KILL cr);
12045
12046 format %{ "salq $dst, $shift" %}
12047 ins_encode %{
12048 __ salq($dst$$Address);
12049 %}
12050 ins_pipe(ialu_mem_reg);
12051 %}
12052
12053 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
12054 %{
12055 predicate(VM_Version::supports_bmi2());
12056 match(Set dst (LShiftL src shift));
12057
12058 format %{ "shlxq $dst, $src, $shift" %}
12059 ins_encode %{
12060 __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
12061 %}
12062 ins_pipe(ialu_reg_reg);
12063 %}
12064
12065 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
12066 %{
12067 predicate(VM_Version::supports_bmi2());
12068 match(Set dst (LShiftL (LoadL src) shift));
12069 ins_cost(175);
12070 format %{ "shlxq $dst, $src, $shift" %}
12071 ins_encode %{
12072 __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
12073 %}
12074 ins_pipe(ialu_reg_mem);
12075 %}
12076
12077 // Arithmetic Shift Right by 8-bit immediate
12078 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
12079 %{
12080 predicate(!UseAPX);
12081 match(Set dst (RShiftL dst shift));
12082 effect(KILL cr);
12083
12084 format %{ "sarq $dst, $shift" %}
12085 ins_encode %{
12086 __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
12087 %}
12088 ins_pipe(ialu_mem_imm);
12089 %}
12090
12091 // Arithmetic Shift Right by 8-bit immediate
12092 instruct sarL_rReg_imm_ndd(rRegL dst, rRegL src, immI shift, rFlagsReg cr)
12093 %{
12094 predicate(UseAPX);
12095 match(Set dst (RShiftL src shift));
12096 effect(KILL cr);
12097 flag(PD::Flag_ndd_demotable_opr1);
12098
12099 format %{ "esarq $dst, $src, $shift\t# long (ndd)" %}
12100 ins_encode %{
12101 __ esarq($dst$$Register, $src$$Register, (unsigned char)($shift$$constant & 0x3F), false);
12102 %}
12103 ins_pipe(ialu_mem_imm);
12104 %}
12105
12106 // Arithmetic Shift Right by 8-bit immediate
12107 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
12108 %{
12109 match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
12110 effect(KILL cr);
12111
12112 format %{ "sarq $dst, $shift" %}
12113 ins_encode %{
12114 __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
12115 %}
12116 ins_pipe(ialu_mem_imm);
12117 %}
12118
12119 // Arithmetic Shift Right by variable
12120 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12121 %{
12122 predicate(!VM_Version::supports_bmi2());
12123 match(Set dst (RShiftL dst shift));
12124 effect(KILL cr);
12125
12126 format %{ "sarq $dst, $shift" %}
12127 ins_encode %{
12128 __ sarq($dst$$Register);
12129 %}
12130 ins_pipe(ialu_reg_reg);
12131 %}
12132
12133 // Arithmetic Shift Right by variable
12134 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
12135 %{
12136 predicate(!VM_Version::supports_bmi2());
12137 match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
12138 effect(KILL cr);
12139
12140 format %{ "sarq $dst, $shift" %}
12141 ins_encode %{
12142 __ sarq($dst$$Address);
12143 %}
12144 ins_pipe(ialu_mem_reg);
12145 %}
12146
12147 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
12148 %{
12149 predicate(VM_Version::supports_bmi2());
12150 match(Set dst (RShiftL src shift));
12151
12152 format %{ "sarxq $dst, $src, $shift" %}
12153 ins_encode %{
12154 __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
12155 %}
12156 ins_pipe(ialu_reg_reg);
12157 %}
12158
12159 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
12160 %{
12161 predicate(VM_Version::supports_bmi2());
12162 match(Set dst (RShiftL (LoadL src) shift));
12163 ins_cost(175);
12164 format %{ "sarxq $dst, $src, $shift" %}
12165 ins_encode %{
12166 __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
12167 %}
12168 ins_pipe(ialu_reg_mem);
12169 %}
12170
12171 // Logical Shift Right by 8-bit immediate
12172 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
12173 %{
12174 predicate(!UseAPX);
12175 match(Set dst (URShiftL dst shift));
12176 effect(KILL cr);
12177
12178 format %{ "shrq $dst, $shift" %}
12179 ins_encode %{
12180 __ shrq($dst$$Register, $shift$$constant);
12181 %}
12182 ins_pipe(ialu_reg);
12183 %}
12184
12185 // Logical Shift Right by 8-bit immediate
12186 instruct shrL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
12187 %{
12188 predicate(UseAPX);
12189 match(Set dst (URShiftL src shift));
12190 effect(KILL cr);
12191 flag(PD::Flag_ndd_demotable_opr1);
12192
12193 format %{ "eshrq $dst, $src, $shift\t# long (ndd)" %}
12194 ins_encode %{
12195 __ eshrq($dst$$Register, $src$$Register, $shift$$constant, false);
12196 %}
12197 ins_pipe(ialu_reg);
12198 %}
12199
12200 // Logical Shift Right by 8-bit immediate
12201 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
12202 %{
12203 match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
12204 effect(KILL cr);
12205
12206 format %{ "shrq $dst, $shift" %}
12207 ins_encode %{
12208 __ shrq($dst$$Address, $shift$$constant);
12209 %}
12210 ins_pipe(ialu_mem_imm);
12211 %}
12212
12213 // Logical Shift Right by variable
12214 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12215 %{
12216 predicate(!VM_Version::supports_bmi2());
12217 match(Set dst (URShiftL dst shift));
12218 effect(KILL cr);
12219
12220 format %{ "shrq $dst, $shift" %}
12221 ins_encode %{
12222 __ shrq($dst$$Register);
12223 %}
12224 ins_pipe(ialu_reg_reg);
12225 %}
12226
12227 // Logical Shift Right by variable
12228 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
12229 %{
12230 predicate(!VM_Version::supports_bmi2());
12231 match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
12232 effect(KILL cr);
12233
12234 format %{ "shrq $dst, $shift" %}
12235 ins_encode %{
12236 __ shrq($dst$$Address);
12237 %}
12238 ins_pipe(ialu_mem_reg);
12239 %}
12240
12241 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
12242 %{
12243 predicate(VM_Version::supports_bmi2());
12244 match(Set dst (URShiftL src shift));
12245
12246 format %{ "shrxq $dst, $src, $shift" %}
12247 ins_encode %{
12248 __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
12249 %}
12250 ins_pipe(ialu_reg_reg);
12251 %}
12252
12253 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
12254 %{
12255 predicate(VM_Version::supports_bmi2());
12256 match(Set dst (URShiftL (LoadL src) shift));
12257 ins_cost(175);
12258 format %{ "shrxq $dst, $src, $shift" %}
12259 ins_encode %{
12260 __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
12261 %}
12262 ins_pipe(ialu_reg_mem);
12263 %}
12264
12265 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
12266 // This idiom is used by the compiler for the i2b bytecode.
12267 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
12268 %{
12269 match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
12270
12271 format %{ "movsbl $dst, $src\t# i2b" %}
12272 ins_encode %{
12273 __ movsbl($dst$$Register, $src$$Register);
12274 %}
12275 ins_pipe(ialu_reg_reg);
12276 %}
12277
12278 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
12279 // This idiom is used by the compiler the i2s bytecode.
12280 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
12281 %{
12282 match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
12283
12284 format %{ "movswl $dst, $src\t# i2s" %}
12285 ins_encode %{
12286 __ movswl($dst$$Register, $src$$Register);
12287 %}
12288 ins_pipe(ialu_reg_reg);
12289 %}
12290
12291 // ROL/ROR instructions
12292
12293 // Rotate left by constant.
12294 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
12295 %{
12296 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12297 match(Set dst (RotateLeft dst shift));
12298 effect(KILL cr);
12299 format %{ "roll $dst, $shift" %}
12300 ins_encode %{
12301 __ roll($dst$$Register, $shift$$constant);
12302 %}
12303 ins_pipe(ialu_reg);
12304 %}
12305
12306 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
12307 %{
12308 predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12309 match(Set dst (RotateLeft src shift));
12310 format %{ "rolxl $dst, $src, $shift" %}
12311 ins_encode %{
12312 int shift = 32 - ($shift$$constant & 31);
12313 __ rorxl($dst$$Register, $src$$Register, shift);
12314 %}
12315 ins_pipe(ialu_reg_reg);
12316 %}
12317
12318 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
12319 %{
12320 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12321 match(Set dst (RotateLeft (LoadI src) shift));
12322 ins_cost(175);
12323 format %{ "rolxl $dst, $src, $shift" %}
12324 ins_encode %{
12325 int shift = 32 - ($shift$$constant & 31);
12326 __ rorxl($dst$$Register, $src$$Address, shift);
12327 %}
12328 ins_pipe(ialu_reg_mem);
12329 %}
12330
12331 // Rotate Left by variable
12332 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
12333 %{
12334 predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
12335 match(Set dst (RotateLeft dst shift));
12336 effect(KILL cr);
12337 format %{ "roll $dst, $shift" %}
12338 ins_encode %{
12339 __ roll($dst$$Register);
12340 %}
12341 ins_pipe(ialu_reg_reg);
12342 %}
12343
12344 // Rotate Left by variable
12345 instruct rolI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
12346 %{
12347 predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
12348 match(Set dst (RotateLeft src shift));
12349 effect(KILL cr);
12350 flag(PD::Flag_ndd_demotable_opr1);
12351
12352 format %{ "eroll $dst, $src, $shift\t# rotate left (int ndd)" %}
12353 ins_encode %{
12354 __ eroll($dst$$Register, $src$$Register, false);
12355 %}
12356 ins_pipe(ialu_reg_reg);
12357 %}
12358
12359 // Rotate Right by constant.
12360 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
12361 %{
12362 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12363 match(Set dst (RotateRight dst shift));
12364 effect(KILL cr);
12365 format %{ "rorl $dst, $shift" %}
12366 ins_encode %{
12367 __ rorl($dst$$Register, $shift$$constant);
12368 %}
12369 ins_pipe(ialu_reg);
12370 %}
12371
12372 // Rotate Right by constant.
12373 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
12374 %{
12375 predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12376 match(Set dst (RotateRight src shift));
12377 format %{ "rorxl $dst, $src, $shift" %}
12378 ins_encode %{
12379 __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
12380 %}
12381 ins_pipe(ialu_reg_reg);
12382 %}
12383
12384 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
12385 %{
12386 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12387 match(Set dst (RotateRight (LoadI src) shift));
12388 ins_cost(175);
12389 format %{ "rorxl $dst, $src, $shift" %}
12390 ins_encode %{
12391 __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
12392 %}
12393 ins_pipe(ialu_reg_mem);
12394 %}
12395
12396 // Rotate Right by variable
12397 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
12398 %{
12399 predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
12400 match(Set dst (RotateRight dst shift));
12401 effect(KILL cr);
12402 format %{ "rorl $dst, $shift" %}
12403 ins_encode %{
12404 __ rorl($dst$$Register);
12405 %}
12406 ins_pipe(ialu_reg_reg);
12407 %}
12408
12409 // Rotate Right by variable
12410 instruct rorI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
12411 %{
12412 predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
12413 match(Set dst (RotateRight src shift));
12414 effect(KILL cr);
12415 flag(PD::Flag_ndd_demotable_opr1);
12416
12417 format %{ "erorl $dst, $src, $shift\t# rotate right(int ndd)" %}
12418 ins_encode %{
12419 __ erorl($dst$$Register, $src$$Register, false);
12420 %}
12421 ins_pipe(ialu_reg_reg);
12422 %}
12423
12424 // Rotate Left by constant.
12425 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
12426 %{
12427 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12428 match(Set dst (RotateLeft dst shift));
12429 effect(KILL cr);
12430 format %{ "rolq $dst, $shift" %}
12431 ins_encode %{
12432 __ rolq($dst$$Register, $shift$$constant);
12433 %}
12434 ins_pipe(ialu_reg);
12435 %}
12436
12437 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
12438 %{
12439 predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12440 match(Set dst (RotateLeft src shift));
12441 format %{ "rolxq $dst, $src, $shift" %}
12442 ins_encode %{
12443 int shift = 64 - ($shift$$constant & 63);
12444 __ rorxq($dst$$Register, $src$$Register, shift);
12445 %}
12446 ins_pipe(ialu_reg_reg);
12447 %}
12448
12449 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
12450 %{
12451 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12452 match(Set dst (RotateLeft (LoadL src) shift));
12453 ins_cost(175);
12454 format %{ "rolxq $dst, $src, $shift" %}
12455 ins_encode %{
12456 int shift = 64 - ($shift$$constant & 63);
12457 __ rorxq($dst$$Register, $src$$Address, shift);
12458 %}
12459 ins_pipe(ialu_reg_mem);
12460 %}
12461
12462 // Rotate Left by variable
12463 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12464 %{
12465 predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
12466 match(Set dst (RotateLeft dst shift));
12467 effect(KILL cr);
12468
12469 format %{ "rolq $dst, $shift" %}
12470 ins_encode %{
12471 __ rolq($dst$$Register);
12472 %}
12473 ins_pipe(ialu_reg_reg);
12474 %}
12475
12476 // Rotate Left by variable
12477 instruct rolL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
12478 %{
12479 predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
12480 match(Set dst (RotateLeft src shift));
12481 effect(KILL cr);
12482 flag(PD::Flag_ndd_demotable_opr1);
12483
12484 format %{ "erolq $dst, $src, $shift\t# rotate left(long ndd)" %}
12485 ins_encode %{
12486 __ erolq($dst$$Register, $src$$Register, false);
12487 %}
12488 ins_pipe(ialu_reg_reg);
12489 %}
12490
12491 // Rotate Right by constant.
12492 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
12493 %{
12494 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12495 match(Set dst (RotateRight dst shift));
12496 effect(KILL cr);
12497 format %{ "rorq $dst, $shift" %}
12498 ins_encode %{
12499 __ rorq($dst$$Register, $shift$$constant);
12500 %}
12501 ins_pipe(ialu_reg);
12502 %}
12503
12504 // Rotate Right by constant
12505 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
12506 %{
12507 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12508 match(Set dst (RotateRight src shift));
12509 format %{ "rorxq $dst, $src, $shift" %}
12510 ins_encode %{
12511 __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
12512 %}
12513 ins_pipe(ialu_reg_reg);
12514 %}
12515
12516 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
12517 %{
12518 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12519 match(Set dst (RotateRight (LoadL src) shift));
12520 ins_cost(175);
12521 format %{ "rorxq $dst, $src, $shift" %}
12522 ins_encode %{
12523 __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
12524 %}
12525 ins_pipe(ialu_reg_mem);
12526 %}
12527
12528 // Rotate Right by variable
12529 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12530 %{
12531 predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
12532 match(Set dst (RotateRight dst shift));
12533 effect(KILL cr);
12534 format %{ "rorq $dst, $shift" %}
12535 ins_encode %{
12536 __ rorq($dst$$Register);
12537 %}
12538 ins_pipe(ialu_reg_reg);
12539 %}
12540
12541 // Rotate Right by variable
12542 instruct rorL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
12543 %{
12544 predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
12545 match(Set dst (RotateRight src shift));
12546 effect(KILL cr);
12547 flag(PD::Flag_ndd_demotable_opr1);
12548
12549 format %{ "erorq $dst, $src, $shift\t# rotate right(long ndd)" %}
12550 ins_encode %{
12551 __ erorq($dst$$Register, $src$$Register, false);
12552 %}
12553 ins_pipe(ialu_reg_reg);
12554 %}
12555
12556 //----------------------------- CompressBits/ExpandBits ------------------------
12557
12558 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
12559 predicate(n->bottom_type()->isa_long());
12560 match(Set dst (CompressBits src mask));
12561 format %{ "pextq $dst, $src, $mask\t! parallel bit extract" %}
12562 ins_encode %{
12563 __ pextq($dst$$Register, $src$$Register, $mask$$Register);
12564 %}
12565 ins_pipe( pipe_slow );
12566 %}
12567
12568 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
12569 predicate(n->bottom_type()->isa_long());
12570 match(Set dst (ExpandBits src mask));
12571 format %{ "pdepq $dst, $src, $mask\t! parallel bit deposit" %}
12572 ins_encode %{
12573 __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
12574 %}
12575 ins_pipe( pipe_slow );
12576 %}
12577
12578 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
12579 predicate(n->bottom_type()->isa_long());
12580 match(Set dst (CompressBits src (LoadL mask)));
12581 format %{ "pextq $dst, $src, $mask\t! parallel bit extract" %}
12582 ins_encode %{
12583 __ pextq($dst$$Register, $src$$Register, $mask$$Address);
12584 %}
12585 ins_pipe( pipe_slow );
12586 %}
12587
12588 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
12589 predicate(n->bottom_type()->isa_long());
12590 match(Set dst (ExpandBits src (LoadL mask)));
12591 format %{ "pdepq $dst, $src, $mask\t! parallel bit deposit" %}
12592 ins_encode %{
12593 __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
12594 %}
12595 ins_pipe( pipe_slow );
12596 %}
12597
12598
12599 // Logical Instructions
12600
12601 // Integer Logical Instructions
12602
12603 // And Instructions
12604 // And Register with Register
12605 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
12606 %{
12607 predicate(!UseAPX);
12608 match(Set dst (AndI dst src));
12609 effect(KILL cr);
12610 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12611
12612 format %{ "andl $dst, $src\t# int" %}
12613 ins_encode %{
12614 __ andl($dst$$Register, $src$$Register);
12615 %}
12616 ins_pipe(ialu_reg_reg);
12617 %}
12618
12619 // And Register with Register using New Data Destination (NDD)
12620 instruct andI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
12621 %{
12622 predicate(UseAPX);
12623 match(Set dst (AndI src1 src2));
12624 effect(KILL cr);
12625 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
12626
12627 format %{ "eandl $dst, $src1, $src2\t# int ndd" %}
12628 ins_encode %{
12629 __ eandl($dst$$Register, $src1$$Register, $src2$$Register, false);
12630
12631 %}
12632 ins_pipe(ialu_reg_reg);
12633 %}
12634
12635 // And Register with Immediate 255
12636 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
12637 %{
12638 match(Set dst (AndI src mask));
12639
12640 format %{ "movzbl $dst, $src\t# int & 0xFF" %}
12641 ins_encode %{
12642 __ movzbl($dst$$Register, $src$$Register);
12643 %}
12644 ins_pipe(ialu_reg);
12645 %}
12646
12647 // And Register with Immediate 255 and promote to long
12648 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
12649 %{
12650 match(Set dst (ConvI2L (AndI src mask)));
12651
12652 format %{ "movzbl $dst, $src\t# int & 0xFF -> long" %}
12653 ins_encode %{
12654 __ movzbl($dst$$Register, $src$$Register);
12655 %}
12656 ins_pipe(ialu_reg);
12657 %}
12658
12659 // And Register with Immediate 65535
12660 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
12661 %{
12662 match(Set dst (AndI src mask));
12663
12664 format %{ "movzwl $dst, $src\t# int & 0xFFFF" %}
12665 ins_encode %{
12666 __ movzwl($dst$$Register, $src$$Register);
12667 %}
12668 ins_pipe(ialu_reg);
12669 %}
12670
12671 // And Register with Immediate 65535 and promote to long
12672 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
12673 %{
12674 match(Set dst (ConvI2L (AndI src mask)));
12675
12676 format %{ "movzwl $dst, $src\t# int & 0xFFFF -> long" %}
12677 ins_encode %{
12678 __ movzwl($dst$$Register, $src$$Register);
12679 %}
12680 ins_pipe(ialu_reg);
12681 %}
12682
12683 // Can skip int2long conversions after AND with small bitmask
12684 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src, immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
12685 %{
12686 predicate(VM_Version::supports_bmi2());
12687 ins_cost(125);
12688 effect(TEMP tmp, KILL cr);
12689 match(Set dst (ConvI2L (AndI src mask)));
12690 format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int & immI_Pow2M1 -> long" %}
12691 ins_encode %{
12692 __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
12693 __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
12694 %}
12695 ins_pipe(ialu_reg_reg);
12696 %}
12697
12698 // And Register with Immediate
12699 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
12700 %{
12701 predicate(!UseAPX);
12702 match(Set dst (AndI dst src));
12703 effect(KILL cr);
12704 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12705
12706 format %{ "andl $dst, $src\t# int" %}
12707 ins_encode %{
12708 __ andl($dst$$Register, $src$$constant);
12709 %}
12710 ins_pipe(ialu_reg);
12711 %}
12712
12713 instruct andI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
12714 %{
12715 predicate(UseAPX);
12716 match(Set dst (AndI src1 src2));
12717 effect(KILL cr);
12718 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
12719
12720 format %{ "eandl $dst, $src1, $src2\t# int ndd" %}
12721 ins_encode %{
12722 __ eandl($dst$$Register, $src1$$Register, $src2$$constant, false);
12723 %}
12724 ins_pipe(ialu_reg);
12725 %}
12726
12727 // And Register with Memory
12728 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
12729 %{
12730 match(Set dst (AndI dst (LoadI src)));
12731 effect(KILL cr);
12732 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12733
12734 ins_cost(150);
12735 format %{ "andl $dst, $src\t# int" %}
12736 ins_encode %{
12737 __ andl($dst$$Register, $src$$Address);
12738 %}
12739 ins_pipe(ialu_reg_mem);
12740 %}
12741
12742 // And Memory with Register
12743 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
12744 %{
12745 match(Set dst (StoreB dst (AndI (LoadB dst) src)));
12746 effect(KILL cr);
12747 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12748
12749 ins_cost(150);
12750 format %{ "andb $dst, $src\t# byte" %}
12751 ins_encode %{
12752 __ andb($dst$$Address, $src$$Register);
12753 %}
12754 ins_pipe(ialu_mem_reg);
12755 %}
12756
12757 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
12758 %{
12759 match(Set dst (StoreI dst (AndI (LoadI dst) src)));
12760 effect(KILL cr);
12761 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12762
12763 ins_cost(150);
12764 format %{ "andl $dst, $src\t# int" %}
12765 ins_encode %{
12766 __ andl($dst$$Address, $src$$Register);
12767 %}
12768 ins_pipe(ialu_mem_reg);
12769 %}
12770
12771 // And Memory with Immediate
12772 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
12773 %{
12774 match(Set dst (StoreI dst (AndI (LoadI dst) src)));
12775 effect(KILL cr);
12776 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12777
12778 ins_cost(125);
12779 format %{ "andl $dst, $src\t# int" %}
12780 ins_encode %{
12781 __ andl($dst$$Address, $src$$constant);
12782 %}
12783 ins_pipe(ialu_mem_imm);
12784 %}
12785
12786 // BMI1 instructions
12787 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
12788 match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
12789 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12790 effect(KILL cr);
12791 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12792
12793 ins_cost(125);
12794 format %{ "andnl $dst, $src1, $src2" %}
12795
12796 ins_encode %{
12797 __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
12798 %}
12799 ins_pipe(ialu_reg_mem);
12800 %}
12801
12802 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
12803 match(Set dst (AndI (XorI src1 minus_1) src2));
12804 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12805 effect(KILL cr);
12806 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12807
12808 format %{ "andnl $dst, $src1, $src2" %}
12809
12810 ins_encode %{
12811 __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
12812 %}
12813 ins_pipe(ialu_reg);
12814 %}
12815
12816 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
12817 match(Set dst (AndI (SubI imm_zero src) src));
12818 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12819 effect(KILL cr);
12820 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12821
12822 format %{ "blsil $dst, $src" %}
12823
12824 ins_encode %{
12825 __ blsil($dst$$Register, $src$$Register);
12826 %}
12827 ins_pipe(ialu_reg);
12828 %}
12829
12830 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
12831 match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
12832 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12833 effect(KILL cr);
12834 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12835
12836 ins_cost(125);
12837 format %{ "blsil $dst, $src" %}
12838
12839 ins_encode %{
12840 __ blsil($dst$$Register, $src$$Address);
12841 %}
12842 ins_pipe(ialu_reg_mem);
12843 %}
12844
12845 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
12846 %{
12847 match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
12848 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12849 effect(KILL cr);
12850 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
12851
12852 ins_cost(125);
12853 format %{ "blsmskl $dst, $src" %}
12854
12855 ins_encode %{
12856 __ blsmskl($dst$$Register, $src$$Address);
12857 %}
12858 ins_pipe(ialu_reg_mem);
12859 %}
12860
12861 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
12862 %{
12863 match(Set dst (XorI (AddI src minus_1) src));
12864 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12865 effect(KILL cr);
12866 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
12867
12868 format %{ "blsmskl $dst, $src" %}
12869
12870 ins_encode %{
12871 __ blsmskl($dst$$Register, $src$$Register);
12872 %}
12873
12874 ins_pipe(ialu_reg);
12875 %}
12876
12877 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
12878 %{
12879 match(Set dst (AndI (AddI src minus_1) src) );
12880 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12881 effect(KILL cr);
12882 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12883
12884 format %{ "blsrl $dst, $src" %}
12885
12886 ins_encode %{
12887 __ blsrl($dst$$Register, $src$$Register);
12888 %}
12889
12890 ins_pipe(ialu_reg_mem);
12891 %}
12892
12893 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
12894 %{
12895 match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
12896 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12897 effect(KILL cr);
12898 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12899
12900 ins_cost(125);
12901 format %{ "blsrl $dst, $src" %}
12902
12903 ins_encode %{
12904 __ blsrl($dst$$Register, $src$$Address);
12905 %}
12906
12907 ins_pipe(ialu_reg);
12908 %}
12909
12910 // Or Instructions
12911 // Or Register with Register
12912 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
12913 %{
12914 predicate(!UseAPX);
12915 match(Set dst (OrI dst src));
12916 effect(KILL cr);
12917 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12918
12919 format %{ "orl $dst, $src\t# int" %}
12920 ins_encode %{
12921 __ orl($dst$$Register, $src$$Register);
12922 %}
12923 ins_pipe(ialu_reg_reg);
12924 %}
12925
12926 // Or Register with Register using New Data Destination (NDD)
12927 instruct orI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
12928 %{
12929 predicate(UseAPX);
12930 match(Set dst (OrI src1 src2));
12931 effect(KILL cr);
12932 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
12933
12934 format %{ "eorl $dst, $src1, $src2\t# int ndd" %}
12935 ins_encode %{
12936 __ eorl($dst$$Register, $src1$$Register, $src2$$Register, false);
12937 %}
12938 ins_pipe(ialu_reg_reg);
12939 %}
12940
12941 // Or Register with Immediate
12942 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
12943 %{
12944 predicate(!UseAPX);
12945 match(Set dst (OrI dst src));
12946 effect(KILL cr);
12947 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12948
12949 format %{ "orl $dst, $src\t# int" %}
12950 ins_encode %{
12951 __ orl($dst$$Register, $src$$constant);
12952 %}
12953 ins_pipe(ialu_reg);
12954 %}
12955
12956 instruct orI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
12957 %{
12958 predicate(UseAPX);
12959 match(Set dst (OrI src1 src2));
12960 effect(KILL cr);
12961 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
12962
12963 format %{ "eorl $dst, $src1, $src2\t# int ndd" %}
12964 ins_encode %{
12965 __ eorl($dst$$Register, $src1$$Register, $src2$$constant, false);
12966 %}
12967 ins_pipe(ialu_reg);
12968 %}
12969
12970 instruct orI_rReg_imm_rReg_ndd(rRegI dst, immI src1, rRegI src2, rFlagsReg cr)
12971 %{
12972 predicate(UseAPX);
12973 match(Set dst (OrI src1 src2));
12974 effect(KILL cr);
12975 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
12976
12977 format %{ "eorl $dst, $src2, $src1\t# int ndd" %}
12978 ins_encode %{
12979 __ eorl($dst$$Register, $src2$$Register, $src1$$constant, false);
12980 %}
12981 ins_pipe(ialu_reg);
12982 %}
12983
12984 // Or Register with Memory
12985 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
12986 %{
12987 match(Set dst (OrI dst (LoadI src)));
12988 effect(KILL cr);
12989 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12990
12991 ins_cost(150);
12992 format %{ "orl $dst, $src\t# int" %}
12993 ins_encode %{
12994 __ orl($dst$$Register, $src$$Address);
12995 %}
12996 ins_pipe(ialu_reg_mem);
12997 %}
12998
12999 // Or Memory with Register
13000 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
13001 %{
13002 match(Set dst (StoreB dst (OrI (LoadB dst) src)));
13003 effect(KILL cr);
13004 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13005
13006 ins_cost(150);
13007 format %{ "orb $dst, $src\t# byte" %}
13008 ins_encode %{
13009 __ orb($dst$$Address, $src$$Register);
13010 %}
13011 ins_pipe(ialu_mem_reg);
13012 %}
13013
13014 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
13015 %{
13016 match(Set dst (StoreI dst (OrI (LoadI dst) src)));
13017 effect(KILL cr);
13018 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13019
13020 ins_cost(150);
13021 format %{ "orl $dst, $src\t# int" %}
13022 ins_encode %{
13023 __ orl($dst$$Address, $src$$Register);
13024 %}
13025 ins_pipe(ialu_mem_reg);
13026 %}
13027
13028 // Or Memory with Immediate
13029 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
13030 %{
13031 match(Set dst (StoreI dst (OrI (LoadI dst) src)));
13032 effect(KILL cr);
13033 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13034
13035 ins_cost(125);
13036 format %{ "orl $dst, $src\t# int" %}
13037 ins_encode %{
13038 __ orl($dst$$Address, $src$$constant);
13039 %}
13040 ins_pipe(ialu_mem_imm);
13041 %}
13042
13043 // Xor Instructions
13044 // Xor Register with Register
13045 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
13046 %{
13047 predicate(!UseAPX);
13048 match(Set dst (XorI dst src));
13049 effect(KILL cr);
13050 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13051
13052 format %{ "xorl $dst, $src\t# int" %}
13053 ins_encode %{
13054 __ xorl($dst$$Register, $src$$Register);
13055 %}
13056 ins_pipe(ialu_reg_reg);
13057 %}
13058
13059 // Xor Register with Register using New Data Destination (NDD)
13060 instruct xorI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13061 %{
13062 predicate(UseAPX);
13063 match(Set dst (XorI src1 src2));
13064 effect(KILL cr);
13065 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13066
13067 format %{ "exorl $dst, $src1, $src2\t# int ndd" %}
13068 ins_encode %{
13069 __ exorl($dst$$Register, $src1$$Register, $src2$$Register, false);
13070 %}
13071 ins_pipe(ialu_reg_reg);
13072 %}
13073
13074 // Xor Register with Immediate -1
13075 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm)
13076 %{
13077 predicate(!UseAPX);
13078 match(Set dst (XorI dst imm));
13079
13080 format %{ "notl $dst" %}
13081 ins_encode %{
13082 __ notl($dst$$Register);
13083 %}
13084 ins_pipe(ialu_reg);
13085 %}
13086
13087 instruct xorI_rReg_im1_ndd(rRegI dst, rRegI src, immI_M1 imm)
13088 %{
13089 match(Set dst (XorI src imm));
13090 predicate(UseAPX);
13091 flag(PD::Flag_ndd_demotable_opr1);
13092
13093 format %{ "enotl $dst, $src" %}
13094 ins_encode %{
13095 __ enotl($dst$$Register, $src$$Register);
13096 %}
13097 ins_pipe(ialu_reg);
13098 %}
13099
13100 // Xor Register with Immediate
13101 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
13102 %{
13103 // Strict predicate check to make selection of xorI_rReg_im1 cost agnostic if immI src is -1.
13104 predicate(!UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
13105 match(Set dst (XorI dst src));
13106 effect(KILL cr);
13107 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13108
13109 format %{ "xorl $dst, $src\t# int" %}
13110 ins_encode %{
13111 __ xorl($dst$$Register, $src$$constant);
13112 %}
13113 ins_pipe(ialu_reg);
13114 %}
13115
13116 instruct xorI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
13117 %{
13118 // Strict predicate check to make selection of xorI_rReg_im1_ndd cost agnostic if immI src2 is -1.
13119 predicate(UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
13120 match(Set dst (XorI src1 src2));
13121 effect(KILL cr);
13122 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13123
13124 format %{ "exorl $dst, $src1, $src2\t# int ndd" %}
13125 ins_encode %{
13126 __ exorl($dst$$Register, $src1$$Register, $src2$$constant, false);
13127 %}
13128 ins_pipe(ialu_reg);
13129 %}
13130
13131 // Xor Register with Memory
13132 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
13133 %{
13134 match(Set dst (XorI dst (LoadI src)));
13135 effect(KILL cr);
13136 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13137
13138 ins_cost(150);
13139 format %{ "xorl $dst, $src\t# int" %}
13140 ins_encode %{
13141 __ xorl($dst$$Register, $src$$Address);
13142 %}
13143 ins_pipe(ialu_reg_mem);
13144 %}
13145
13146 // Xor Memory with Register
13147 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
13148 %{
13149 match(Set dst (StoreB dst (XorI (LoadB dst) src)));
13150 effect(KILL cr);
13151 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13152
13153 ins_cost(150);
13154 format %{ "xorb $dst, $src\t# byte" %}
13155 ins_encode %{
13156 __ xorb($dst$$Address, $src$$Register);
13157 %}
13158 ins_pipe(ialu_mem_reg);
13159 %}
13160
13161 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
13162 %{
13163 match(Set dst (StoreI dst (XorI (LoadI dst) src)));
13164 effect(KILL cr);
13165 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13166
13167 ins_cost(150);
13168 format %{ "xorl $dst, $src\t# int" %}
13169 ins_encode %{
13170 __ xorl($dst$$Address, $src$$Register);
13171 %}
13172 ins_pipe(ialu_mem_reg);
13173 %}
13174
13175 // Xor Memory with Immediate
13176 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
13177 %{
13178 match(Set dst (StoreI dst (XorI (LoadI dst) src)));
13179 effect(KILL cr);
13180 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13181
13182 ins_cost(125);
13183 format %{ "xorl $dst, $src\t# int" %}
13184 ins_encode %{
13185 __ xorl($dst$$Address, $src$$constant);
13186 %}
13187 ins_pipe(ialu_mem_imm);
13188 %}
13189
13190
13191 // Long Logical Instructions
13192
13193 // And Instructions
13194 // And Register with Register
13195 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
13196 %{
13197 predicate(!UseAPX);
13198 match(Set dst (AndL dst src));
13199 effect(KILL cr);
13200 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13201
13202 format %{ "andq $dst, $src\t# long" %}
13203 ins_encode %{
13204 __ andq($dst$$Register, $src$$Register);
13205 %}
13206 ins_pipe(ialu_reg_reg);
13207 %}
13208
13209 // And Register with Register using New Data Destination (NDD)
13210 instruct andL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
13211 %{
13212 predicate(UseAPX);
13213 match(Set dst (AndL src1 src2));
13214 effect(KILL cr);
13215 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13216
13217 format %{ "eandq $dst, $src1, $src2\t# long ndd" %}
13218 ins_encode %{
13219 __ eandq($dst$$Register, $src1$$Register, $src2$$Register, false);
13220
13221 %}
13222 ins_pipe(ialu_reg_reg);
13223 %}
13224
13225 // And Register with Immediate 255
13226 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
13227 %{
13228 match(Set dst (AndL src mask));
13229
13230 format %{ "movzbl $dst, $src\t# long & 0xFF" %}
13231 ins_encode %{
13232 // movzbl zeroes out the upper 32-bit and does not need REX.W
13233 __ movzbl($dst$$Register, $src$$Register);
13234 %}
13235 ins_pipe(ialu_reg);
13236 %}
13237
13238 // And Register with Immediate 65535
13239 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
13240 %{
13241 match(Set dst (AndL src mask));
13242
13243 format %{ "movzwl $dst, $src\t# long & 0xFFFF" %}
13244 ins_encode %{
13245 // movzwl zeroes out the upper 32-bit and does not need REX.W
13246 __ movzwl($dst$$Register, $src$$Register);
13247 %}
13248 ins_pipe(ialu_reg);
13249 %}
13250
13251 // And Register with Immediate
13252 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
13253 %{
13254 predicate(!UseAPX);
13255 match(Set dst (AndL dst src));
13256 effect(KILL cr);
13257 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13258
13259 format %{ "andq $dst, $src\t# long" %}
13260 ins_encode %{
13261 __ andq($dst$$Register, $src$$constant);
13262 %}
13263 ins_pipe(ialu_reg);
13264 %}
13265
13266 instruct andL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
13267 %{
13268 predicate(UseAPX);
13269 match(Set dst (AndL src1 src2));
13270 effect(KILL cr);
13271 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13272
13273 format %{ "eandq $dst, $src1, $src2\t# long ndd" %}
13274 ins_encode %{
13275 __ eandq($dst$$Register, $src1$$Register, $src2$$constant, false);
13276 %}
13277 ins_pipe(ialu_reg);
13278 %}
13279
13280 // And Register with Memory
13281 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
13282 %{
13283 match(Set dst (AndL dst (LoadL src)));
13284 effect(KILL cr);
13285 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13286
13287 ins_cost(150);
13288 format %{ "andq $dst, $src\t# long" %}
13289 ins_encode %{
13290 __ andq($dst$$Register, $src$$Address);
13291 %}
13292 ins_pipe(ialu_reg_mem);
13293 %}
13294
13295 // And Memory with Register
13296 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
13297 %{
13298 match(Set dst (StoreL dst (AndL (LoadL dst) src)));
13299 effect(KILL cr);
13300 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13301
13302 ins_cost(150);
13303 format %{ "andq $dst, $src\t# long" %}
13304 ins_encode %{
13305 __ andq($dst$$Address, $src$$Register);
13306 %}
13307 ins_pipe(ialu_mem_reg);
13308 %}
13309
13310 // And Memory with Immediate
13311 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
13312 %{
13313 match(Set dst (StoreL dst (AndL (LoadL dst) src)));
13314 effect(KILL cr);
13315 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13316
13317 ins_cost(125);
13318 format %{ "andq $dst, $src\t# long" %}
13319 ins_encode %{
13320 __ andq($dst$$Address, $src$$constant);
13321 %}
13322 ins_pipe(ialu_mem_imm);
13323 %}
13324
13325 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
13326 %{
13327 // con should be a pure 64-bit immediate given that not(con) is a power of 2
13328 // because AND/OR works well enough for 8/32-bit values.
13329 predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
13330
13331 match(Set dst (StoreL dst (AndL (LoadL dst) con)));
13332 effect(KILL cr);
13333
13334 ins_cost(125);
13335 format %{ "btrq $dst, log2(not($con))\t# long" %}
13336 ins_encode %{
13337 __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
13338 %}
13339 ins_pipe(ialu_mem_imm);
13340 %}
13341
13342 // BMI1 instructions
13343 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
13344 match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
13345 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13346 effect(KILL cr);
13347 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13348
13349 ins_cost(125);
13350 format %{ "andnq $dst, $src1, $src2" %}
13351
13352 ins_encode %{
13353 __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
13354 %}
13355 ins_pipe(ialu_reg_mem);
13356 %}
13357
13358 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
13359 match(Set dst (AndL (XorL src1 minus_1) src2));
13360 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13361 effect(KILL cr);
13362 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13363
13364 format %{ "andnq $dst, $src1, $src2" %}
13365
13366 ins_encode %{
13367 __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
13368 %}
13369 ins_pipe(ialu_reg_mem);
13370 %}
13371
13372 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
13373 match(Set dst (AndL (SubL imm_zero src) src));
13374 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13375 effect(KILL cr);
13376 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13377
13378 format %{ "blsiq $dst, $src" %}
13379
13380 ins_encode %{
13381 __ blsiq($dst$$Register, $src$$Register);
13382 %}
13383 ins_pipe(ialu_reg);
13384 %}
13385
13386 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
13387 match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
13388 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13389 effect(KILL cr);
13390 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13391
13392 ins_cost(125);
13393 format %{ "blsiq $dst, $src" %}
13394
13395 ins_encode %{
13396 __ blsiq($dst$$Register, $src$$Address);
13397 %}
13398 ins_pipe(ialu_reg_mem);
13399 %}
13400
13401 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
13402 %{
13403 match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
13404 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13405 effect(KILL cr);
13406 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
13407
13408 ins_cost(125);
13409 format %{ "blsmskq $dst, $src" %}
13410
13411 ins_encode %{
13412 __ blsmskq($dst$$Register, $src$$Address);
13413 %}
13414 ins_pipe(ialu_reg_mem);
13415 %}
13416
13417 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
13418 %{
13419 match(Set dst (XorL (AddL src minus_1) src));
13420 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13421 effect(KILL cr);
13422 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
13423
13424 format %{ "blsmskq $dst, $src" %}
13425
13426 ins_encode %{
13427 __ blsmskq($dst$$Register, $src$$Register);
13428 %}
13429
13430 ins_pipe(ialu_reg);
13431 %}
13432
13433 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
13434 %{
13435 match(Set dst (AndL (AddL src minus_1) src) );
13436 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13437 effect(KILL cr);
13438 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13439
13440 format %{ "blsrq $dst, $src" %}
13441
13442 ins_encode %{
13443 __ blsrq($dst$$Register, $src$$Register);
13444 %}
13445
13446 ins_pipe(ialu_reg);
13447 %}
13448
13449 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
13450 %{
13451 match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
13452 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13453 effect(KILL cr);
13454 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13455
13456 ins_cost(125);
13457 format %{ "blsrq $dst, $src" %}
13458
13459 ins_encode %{
13460 __ blsrq($dst$$Register, $src$$Address);
13461 %}
13462
13463 ins_pipe(ialu_reg);
13464 %}
13465
13466 // Or Instructions
13467 // Or Register with Register
13468 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
13469 %{
13470 predicate(!UseAPX);
13471 match(Set dst (OrL dst src));
13472 effect(KILL cr);
13473 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13474
13475 format %{ "orq $dst, $src\t# long" %}
13476 ins_encode %{
13477 __ orq($dst$$Register, $src$$Register);
13478 %}
13479 ins_pipe(ialu_reg_reg);
13480 %}
13481
13482 // Or Register with Register using New Data Destination (NDD)
13483 instruct orL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
13484 %{
13485 predicate(UseAPX);
13486 match(Set dst (OrL src1 src2));
13487 effect(KILL cr);
13488 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13489
13490 format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
13491 ins_encode %{
13492 __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
13493
13494 %}
13495 ins_pipe(ialu_reg_reg);
13496 %}
13497
13498 // Use any_RegP to match R15 (TLS register) without spilling.
13499 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
13500 predicate(!UseAPX);
13501 match(Set dst (OrL dst (CastP2X src)));
13502 effect(KILL cr);
13503 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13504
13505 format %{ "orq $dst, $src\t# long" %}
13506 ins_encode %{
13507 __ orq($dst$$Register, $src$$Register);
13508 %}
13509 ins_pipe(ialu_reg_reg);
13510 %}
13511
13512 instruct orL_rReg_castP2X_ndd(rRegL dst, any_RegP src1, any_RegP src2, rFlagsReg cr) %{
13513 predicate(UseAPX);
13514 match(Set dst (OrL src1 (CastP2X src2)));
13515 effect(KILL cr);
13516 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13517
13518 format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
13519 ins_encode %{
13520 __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
13521 %}
13522 ins_pipe(ialu_reg_reg);
13523 %}
13524
13525 // Or Register with Immediate
13526 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
13527 %{
13528 predicate(!UseAPX);
13529 match(Set dst (OrL dst src));
13530 effect(KILL cr);
13531 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13532
13533 format %{ "orq $dst, $src\t# long" %}
13534 ins_encode %{
13535 __ orq($dst$$Register, $src$$constant);
13536 %}
13537 ins_pipe(ialu_reg);
13538 %}
13539
13540 instruct orL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
13541 %{
13542 predicate(UseAPX);
13543 match(Set dst (OrL src1 src2));
13544 effect(KILL cr);
13545 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13546
13547 format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
13548 ins_encode %{
13549 __ eorq($dst$$Register, $src1$$Register, $src2$$constant, false);
13550 %}
13551 ins_pipe(ialu_reg);
13552 %}
13553
13554 instruct orL_rReg_imm_rReg_ndd(rRegL dst, immL32 src1, rRegL src2, rFlagsReg cr)
13555 %{
13556 predicate(UseAPX);
13557 match(Set dst (OrL src1 src2));
13558 effect(KILL cr);
13559 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13560
13561 format %{ "eorq $dst, $src2, $src1\t# long ndd" %}
13562 ins_encode %{
13563 __ eorq($dst$$Register, $src2$$Register, $src1$$constant, false);
13564 %}
13565 ins_pipe(ialu_reg);
13566 %}
13567
13568 // Or Register with Memory
13569 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
13570 %{
13571 match(Set dst (OrL dst (LoadL src)));
13572 effect(KILL cr);
13573 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13574
13575 ins_cost(150);
13576 format %{ "orq $dst, $src\t# long" %}
13577 ins_encode %{
13578 __ orq($dst$$Register, $src$$Address);
13579 %}
13580 ins_pipe(ialu_reg_mem);
13581 %}
13582
13583 // Or Memory with Register
13584 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
13585 %{
13586 match(Set dst (StoreL dst (OrL (LoadL dst) src)));
13587 effect(KILL cr);
13588 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13589
13590 ins_cost(150);
13591 format %{ "orq $dst, $src\t# long" %}
13592 ins_encode %{
13593 __ orq($dst$$Address, $src$$Register);
13594 %}
13595 ins_pipe(ialu_mem_reg);
13596 %}
13597
13598 // Or Memory with Immediate
13599 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
13600 %{
13601 match(Set dst (StoreL dst (OrL (LoadL dst) src)));
13602 effect(KILL cr);
13603 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13604
13605 ins_cost(125);
13606 format %{ "orq $dst, $src\t# long" %}
13607 ins_encode %{
13608 __ orq($dst$$Address, $src$$constant);
13609 %}
13610 ins_pipe(ialu_mem_imm);
13611 %}
13612
13613 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
13614 %{
13615 // con should be a pure 64-bit power of 2 immediate
13616 // because AND/OR works well enough for 8/32-bit values.
13617 predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
13618
13619 match(Set dst (StoreL dst (OrL (LoadL dst) con)));
13620 effect(KILL cr);
13621
13622 ins_cost(125);
13623 format %{ "btsq $dst, log2($con)\t# long" %}
13624 ins_encode %{
13625 __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
13626 %}
13627 ins_pipe(ialu_mem_imm);
13628 %}
13629
13630 // Xor Instructions
13631 // Xor Register with Register
13632 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
13633 %{
13634 predicate(!UseAPX);
13635 match(Set dst (XorL dst src));
13636 effect(KILL cr);
13637 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13638
13639 format %{ "xorq $dst, $src\t# long" %}
13640 ins_encode %{
13641 __ xorq($dst$$Register, $src$$Register);
13642 %}
13643 ins_pipe(ialu_reg_reg);
13644 %}
13645
13646 // Xor Register with Register using New Data Destination (NDD)
13647 instruct xorL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
13648 %{
13649 predicate(UseAPX);
13650 match(Set dst (XorL src1 src2));
13651 effect(KILL cr);
13652 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13653
13654 format %{ "exorq $dst, $src1, $src2\t# long ndd" %}
13655 ins_encode %{
13656 __ exorq($dst$$Register, $src1$$Register, $src2$$Register, false);
13657 %}
13658 ins_pipe(ialu_reg_reg);
13659 %}
13660
13661 // Xor Register with Immediate -1
13662 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm)
13663 %{
13664 predicate(!UseAPX);
13665 match(Set dst (XorL dst imm));
13666
13667 format %{ "notq $dst" %}
13668 ins_encode %{
13669 __ notq($dst$$Register);
13670 %}
13671 ins_pipe(ialu_reg);
13672 %}
13673
13674 instruct xorL_rReg_im1_ndd(rRegL dst,rRegL src, immL_M1 imm)
13675 %{
13676 predicate(UseAPX);
13677 match(Set dst (XorL src imm));
13678 flag(PD::Flag_ndd_demotable_opr1);
13679
13680 format %{ "enotq $dst, $src" %}
13681 ins_encode %{
13682 __ enotq($dst$$Register, $src$$Register);
13683 %}
13684 ins_pipe(ialu_reg);
13685 %}
13686
13687 // Xor Register with Immediate
13688 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
13689 %{
13690 // Strict predicate check to make selection of xorL_rReg_im1 cost agnostic if immL32 src is -1.
13691 predicate(!UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
13692 match(Set dst (XorL dst src));
13693 effect(KILL cr);
13694 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13695
13696 format %{ "xorq $dst, $src\t# long" %}
13697 ins_encode %{
13698 __ xorq($dst$$Register, $src$$constant);
13699 %}
13700 ins_pipe(ialu_reg);
13701 %}
13702
13703 instruct xorL_rReg_rReg_imm(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
13704 %{
13705 // Strict predicate check to make selection of xorL_rReg_im1_ndd cost agnostic if immL32 src2 is -1.
13706 predicate(UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
13707 match(Set dst (XorL src1 src2));
13708 effect(KILL cr);
13709 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13710
13711 format %{ "exorq $dst, $src1, $src2\t# long ndd" %}
13712 ins_encode %{
13713 __ exorq($dst$$Register, $src1$$Register, $src2$$constant, false);
13714 %}
13715 ins_pipe(ialu_reg);
13716 %}
13717
13718 // Xor Register with Memory
13719 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
13720 %{
13721 match(Set dst (XorL dst (LoadL src)));
13722 effect(KILL cr);
13723 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13724
13725 ins_cost(150);
13726 format %{ "xorq $dst, $src\t# long" %}
13727 ins_encode %{
13728 __ xorq($dst$$Register, $src$$Address);
13729 %}
13730 ins_pipe(ialu_reg_mem);
13731 %}
13732
13733 // Xor Memory with Register
13734 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
13735 %{
13736 match(Set dst (StoreL dst (XorL (LoadL dst) src)));
13737 effect(KILL cr);
13738 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13739
13740 ins_cost(150);
13741 format %{ "xorq $dst, $src\t# long" %}
13742 ins_encode %{
13743 __ xorq($dst$$Address, $src$$Register);
13744 %}
13745 ins_pipe(ialu_mem_reg);
13746 %}
13747
13748 // Xor Memory with Immediate
13749 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
13750 %{
13751 match(Set dst (StoreL dst (XorL (LoadL dst) src)));
13752 effect(KILL cr);
13753 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13754
13755 ins_cost(125);
13756 format %{ "xorq $dst, $src\t# long" %}
13757 ins_encode %{
13758 __ xorq($dst$$Address, $src$$constant);
13759 %}
13760 ins_pipe(ialu_mem_imm);
13761 %}
13762
13763 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
13764 %{
13765 match(Set dst (CmpLTMask p q));
13766 effect(KILL cr);
13767
13768 ins_cost(400);
13769 format %{ "cmpl $p, $q\t# cmpLTMask\n\t"
13770 "setcc $dst \t# emits setlt + movzbl or setzul for APX"
13771 "negl $dst" %}
13772 ins_encode %{
13773 __ cmpl($p$$Register, $q$$Register);
13774 __ setcc(Assembler::less, $dst$$Register);
13775 __ negl($dst$$Register);
13776 %}
13777 ins_pipe(pipe_slow);
13778 %}
13779
13780 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
13781 %{
13782 match(Set dst (CmpLTMask dst zero));
13783 effect(KILL cr);
13784
13785 ins_cost(100);
13786 format %{ "sarl $dst, #31\t# cmpLTMask0" %}
13787 ins_encode %{
13788 __ sarl($dst$$Register, 31);
13789 %}
13790 ins_pipe(ialu_reg);
13791 %}
13792
13793 /* Better to save a register than avoid a branch */
13794 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
13795 %{
13796 match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
13797 effect(KILL cr);
13798 ins_cost(300);
13799 format %{ "subl $p,$q\t# cadd_cmpLTMask\n\t"
13800 "jge done\n\t"
13801 "addl $p,$y\n"
13802 "done: " %}
13803 ins_encode %{
13804 Register Rp = $p$$Register;
13805 Register Rq = $q$$Register;
13806 Register Ry = $y$$Register;
13807 Label done;
13808 __ subl(Rp, Rq);
13809 __ jccb(Assembler::greaterEqual, done);
13810 __ addl(Rp, Ry);
13811 __ bind(done);
13812 %}
13813 ins_pipe(pipe_cmplt);
13814 %}
13815
13816 /* Better to save a register than avoid a branch */
13817 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
13818 %{
13819 match(Set y (AndI (CmpLTMask p q) y));
13820 effect(KILL cr);
13821
13822 ins_cost(300);
13823
13824 format %{ "cmpl $p, $q\t# and_cmpLTMask\n\t"
13825 "jlt done\n\t"
13826 "xorl $y, $y\n"
13827 "done: " %}
13828 ins_encode %{
13829 Register Rp = $p$$Register;
13830 Register Rq = $q$$Register;
13831 Register Ry = $y$$Register;
13832 Label done;
13833 __ cmpl(Rp, Rq);
13834 __ jccb(Assembler::less, done);
13835 __ xorl(Ry, Ry);
13836 __ bind(done);
13837 %}
13838 ins_pipe(pipe_cmplt);
13839 %}
13840
13841
13842 //---------- FP Instructions------------------------------------------------
13843
13844 // Really expensive, avoid
13845 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
13846 %{
13847 match(Set cr (CmpF src1 src2));
13848
13849 ins_cost(500);
13850 format %{ "ucomiss $src1, $src2\n\t"
13851 "jnp,s exit\n\t"
13852 "pushfq\t# saw NaN, set CF\n\t"
13853 "andq [rsp], #0xffffff2b\n\t"
13854 "popfq\n"
13855 "exit:" %}
13856 ins_encode %{
13857 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
13858 emit_cmpfp_fixup(masm);
13859 %}
13860 ins_pipe(pipe_slow);
13861 %}
13862
13863 instruct cmpF_cc_regCF(rFlagsRegUCF cr, regF src1, regF src2) %{
13864 match(Set cr (CmpF src1 src2));
13865
13866 ins_cost(100);
13867 format %{ "ucomiss $src1, $src2" %}
13868 ins_encode %{
13869 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
13870 %}
13871 ins_pipe(pipe_slow);
13872 %}
13873
13874 instruct cmpF_cc_regCFE(rFlagsRegUCFE cr, regF src1, regF src2) %{
13875 match(Set cr (CmpF src1 src2));
13876
13877 ins_cost(100);
13878 format %{ "evucomxss $src1, $src2" %}
13879 ins_encode %{
13880 __ evucomxss($src1$$XMMRegister, $src2$$XMMRegister);
13881 %}
13882 ins_pipe(pipe_slow);
13883 %}
13884
13885 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
13886 match(Set cr (CmpF src1 (LoadF src2)));
13887
13888 ins_cost(100);
13889 format %{ "ucomiss $src1, $src2" %}
13890 ins_encode %{
13891 __ ucomiss($src1$$XMMRegister, $src2$$Address);
13892 %}
13893 ins_pipe(pipe_slow);
13894 %}
13895
13896 instruct cmpF_cc_memCFE(rFlagsRegUCFE cr, regF src1, memory src2) %{
13897 match(Set cr (CmpF src1 (LoadF src2)));
13898
13899 ins_cost(100);
13900 format %{ "evucomxss $src1, $src2" %}
13901 ins_encode %{
13902 __ evucomxss($src1$$XMMRegister, $src2$$Address);
13903 %}
13904 ins_pipe(pipe_slow);
13905 %}
13906
13907 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
13908 match(Set cr (CmpF src con));
13909
13910 ins_cost(100);
13911 format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
13912 ins_encode %{
13913 __ ucomiss($src$$XMMRegister, $constantaddress($con));
13914 %}
13915 ins_pipe(pipe_slow);
13916 %}
13917
13918 instruct cmpF_cc_immCFE(rFlagsRegUCFE cr, regF src, immF con) %{
13919 match(Set cr (CmpF src con));
13920
13921 ins_cost(100);
13922 format %{ "evucomxss $src, [$constantaddress]\t# load from constant table: float=$con" %}
13923 ins_encode %{
13924 __ evucomxss($src$$XMMRegister, $constantaddress($con));
13925 %}
13926 ins_pipe(pipe_slow);
13927 %}
13928
13929 // Really expensive, avoid
13930 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
13931 %{
13932 match(Set cr (CmpD src1 src2));
13933
13934 ins_cost(500);
13935 format %{ "ucomisd $src1, $src2\n\t"
13936 "jnp,s exit\n\t"
13937 "pushfq\t# saw NaN, set CF\n\t"
13938 "andq [rsp], #0xffffff2b\n\t"
13939 "popfq\n"
13940 "exit:" %}
13941 ins_encode %{
13942 __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
13943 emit_cmpfp_fixup(masm);
13944 %}
13945 ins_pipe(pipe_slow);
13946 %}
13947
13948 instruct cmpD_cc_regCF(rFlagsRegUCF cr, regD src1, regD src2) %{
13949 match(Set cr (CmpD src1 src2));
13950
13951 ins_cost(100);
13952 format %{ "ucomisd $src1, $src2 test" %}
13953 ins_encode %{
13954 __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
13955 %}
13956 ins_pipe(pipe_slow);
13957 %}
13958
13959 instruct cmpD_cc_regCFE(rFlagsRegUCFE cr, regD src1, regD src2) %{
13960 match(Set cr (CmpD src1 src2));
13961
13962 ins_cost(100);
13963 format %{ "evucomxsd $src1, $src2 test" %}
13964 ins_encode %{
13965 __ evucomxsd($src1$$XMMRegister, $src2$$XMMRegister);
13966 %}
13967 ins_pipe(pipe_slow);
13968 %}
13969
13970 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
13971 match(Set cr (CmpD src1 (LoadD src2)));
13972
13973 ins_cost(100);
13974 format %{ "ucomisd $src1, $src2" %}
13975 ins_encode %{
13976 __ ucomisd($src1$$XMMRegister, $src2$$Address);
13977 %}
13978 ins_pipe(pipe_slow);
13979 %}
13980
13981 instruct cmpD_cc_memCFE(rFlagsRegUCFE cr, regD src1, memory src2) %{
13982 match(Set cr (CmpD src1 (LoadD src2)));
13983
13984 ins_cost(100);
13985 format %{ "evucomxsd $src1, $src2" %}
13986 ins_encode %{
13987 __ evucomxsd($src1$$XMMRegister, $src2$$Address);
13988 %}
13989 ins_pipe(pipe_slow);
13990 %}
13991
13992 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
13993 match(Set cr (CmpD src con));
13994 ins_cost(100);
13995 format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
13996 ins_encode %{
13997 __ ucomisd($src$$XMMRegister, $constantaddress($con));
13998 %}
13999 ins_pipe(pipe_slow);
14000 %}
14001
14002 instruct cmpD_cc_immCFE(rFlagsRegUCFE cr, regD src, immD con) %{
14003 match(Set cr (CmpD src con));
14004
14005 ins_cost(100);
14006 format %{ "evucomxsd $src, [$constantaddress]\t# load from constant table: double=$con" %}
14007 ins_encode %{
14008 __ evucomxsd($src$$XMMRegister, $constantaddress($con));
14009 %}
14010 ins_pipe(pipe_slow);
14011 %}
14012
14013 // Compare into -1,0,1
14014 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
14015 %{
14016 match(Set dst (CmpF3 src1 src2));
14017 effect(KILL cr);
14018
14019 ins_cost(275);
14020 format %{ "ucomiss $src1, $src2\n\t"
14021 "movl $dst, #-1\n\t"
14022 "jp,s done\n\t"
14023 "jb,s done\n\t"
14024 "setne $dst\n\t"
14025 "movzbl $dst, $dst\n"
14026 "done:" %}
14027 ins_encode %{
14028 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
14029 emit_cmpfp3(masm, $dst$$Register);
14030 %}
14031 ins_pipe(pipe_slow);
14032 %}
14033
14034 // Compare into -1,0,1
14035 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
14036 %{
14037 match(Set dst (CmpF3 src1 (LoadF src2)));
14038 effect(KILL cr);
14039
14040 ins_cost(275);
14041 format %{ "ucomiss $src1, $src2\n\t"
14042 "movl $dst, #-1\n\t"
14043 "jp,s done\n\t"
14044 "jb,s done\n\t"
14045 "setne $dst\n\t"
14046 "movzbl $dst, $dst\n"
14047 "done:" %}
14048 ins_encode %{
14049 __ ucomiss($src1$$XMMRegister, $src2$$Address);
14050 emit_cmpfp3(masm, $dst$$Register);
14051 %}
14052 ins_pipe(pipe_slow);
14053 %}
14054
14055 // Compare into -1,0,1
14056 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
14057 match(Set dst (CmpF3 src con));
14058 effect(KILL cr);
14059
14060 ins_cost(275);
14061 format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
14062 "movl $dst, #-1\n\t"
14063 "jp,s done\n\t"
14064 "jb,s done\n\t"
14065 "setne $dst\n\t"
14066 "movzbl $dst, $dst\n"
14067 "done:" %}
14068 ins_encode %{
14069 __ ucomiss($src$$XMMRegister, $constantaddress($con));
14070 emit_cmpfp3(masm, $dst$$Register);
14071 %}
14072 ins_pipe(pipe_slow);
14073 %}
14074
14075 // Compare into -1,0,1
14076 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
14077 %{
14078 match(Set dst (CmpD3 src1 src2));
14079 effect(KILL cr);
14080
14081 ins_cost(275);
14082 format %{ "ucomisd $src1, $src2\n\t"
14083 "movl $dst, #-1\n\t"
14084 "jp,s done\n\t"
14085 "jb,s done\n\t"
14086 "setne $dst\n\t"
14087 "movzbl $dst, $dst\n"
14088 "done:" %}
14089 ins_encode %{
14090 __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
14091 emit_cmpfp3(masm, $dst$$Register);
14092 %}
14093 ins_pipe(pipe_slow);
14094 %}
14095
14096 // Compare into -1,0,1
14097 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
14098 %{
14099 match(Set dst (CmpD3 src1 (LoadD src2)));
14100 effect(KILL cr);
14101
14102 ins_cost(275);
14103 format %{ "ucomisd $src1, $src2\n\t"
14104 "movl $dst, #-1\n\t"
14105 "jp,s done\n\t"
14106 "jb,s done\n\t"
14107 "setne $dst\n\t"
14108 "movzbl $dst, $dst\n"
14109 "done:" %}
14110 ins_encode %{
14111 __ ucomisd($src1$$XMMRegister, $src2$$Address);
14112 emit_cmpfp3(masm, $dst$$Register);
14113 %}
14114 ins_pipe(pipe_slow);
14115 %}
14116
14117 // Compare into -1,0,1
14118 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
14119 match(Set dst (CmpD3 src con));
14120 effect(KILL cr);
14121
14122 ins_cost(275);
14123 format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
14124 "movl $dst, #-1\n\t"
14125 "jp,s done\n\t"
14126 "jb,s done\n\t"
14127 "setne $dst\n\t"
14128 "movzbl $dst, $dst\n"
14129 "done:" %}
14130 ins_encode %{
14131 __ ucomisd($src$$XMMRegister, $constantaddress($con));
14132 emit_cmpfp3(masm, $dst$$Register);
14133 %}
14134 ins_pipe(pipe_slow);
14135 %}
14136
14137 //----------Arithmetic Conversion Instructions---------------------------------
14138
14139 instruct convF2D_reg_reg(regD dst, regF src)
14140 %{
14141 match(Set dst (ConvF2D src));
14142
14143 format %{ "cvtss2sd $dst, $src" %}
14144 ins_encode %{
14145 __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
14146 %}
14147 ins_pipe(pipe_slow); // XXX
14148 %}
14149
14150 instruct convF2D_reg_mem(regD dst, memory src)
14151 %{
14152 predicate(UseAVX == 0);
14153 match(Set dst (ConvF2D (LoadF src)));
14154
14155 format %{ "cvtss2sd $dst, $src" %}
14156 ins_encode %{
14157 __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
14158 %}
14159 ins_pipe(pipe_slow); // XXX
14160 %}
14161
14162 instruct convD2F_reg_reg(regF dst, regD src)
14163 %{
14164 match(Set dst (ConvD2F src));
14165
14166 format %{ "cvtsd2ss $dst, $src" %}
14167 ins_encode %{
14168 __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
14169 %}
14170 ins_pipe(pipe_slow); // XXX
14171 %}
14172
14173 instruct convD2F_reg_mem(regF dst, memory src)
14174 %{
14175 predicate(UseAVX == 0);
14176 match(Set dst (ConvD2F (LoadD src)));
14177
14178 format %{ "cvtsd2ss $dst, $src" %}
14179 ins_encode %{
14180 __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
14181 %}
14182 ins_pipe(pipe_slow); // XXX
14183 %}
14184
14185 // XXX do mem variants
14186 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
14187 %{
14188 predicate(!VM_Version::supports_avx10_2());
14189 match(Set dst (ConvF2I src));
14190 effect(KILL cr);
14191 format %{ "convert_f2i $dst, $src" %}
14192 ins_encode %{
14193 __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
14194 %}
14195 ins_pipe(pipe_slow);
14196 %}
14197
14198 instruct convF2I_reg_reg_avx10_2(rRegI dst, regF src)
14199 %{
14200 predicate(VM_Version::supports_avx10_2());
14201 match(Set dst (ConvF2I src));
14202 format %{ "evcvttss2sisl $dst, $src" %}
14203 ins_encode %{
14204 __ evcvttss2sisl($dst$$Register, $src$$XMMRegister);
14205 %}
14206 ins_pipe(pipe_slow);
14207 %}
14208
14209 instruct convF2I_reg_mem_avx10_2(rRegI dst, memory src)
14210 %{
14211 predicate(VM_Version::supports_avx10_2());
14212 match(Set dst (ConvF2I (LoadF src)));
14213 format %{ "evcvttss2sisl $dst, $src" %}
14214 ins_encode %{
14215 __ evcvttss2sisl($dst$$Register, $src$$Address);
14216 %}
14217 ins_pipe(pipe_slow);
14218 %}
14219
14220 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
14221 %{
14222 predicate(!VM_Version::supports_avx10_2());
14223 match(Set dst (ConvF2L src));
14224 effect(KILL cr);
14225 format %{ "convert_f2l $dst, $src"%}
14226 ins_encode %{
14227 __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
14228 %}
14229 ins_pipe(pipe_slow);
14230 %}
14231
14232 instruct convF2L_reg_reg_avx10_2(rRegL dst, regF src)
14233 %{
14234 predicate(VM_Version::supports_avx10_2());
14235 match(Set dst (ConvF2L src));
14236 format %{ "evcvttss2sisq $dst, $src" %}
14237 ins_encode %{
14238 __ evcvttss2sisq($dst$$Register, $src$$XMMRegister);
14239 %}
14240 ins_pipe(pipe_slow);
14241 %}
14242
14243 instruct convF2L_reg_mem_avx10_2(rRegL dst, memory src)
14244 %{
14245 predicate(VM_Version::supports_avx10_2());
14246 match(Set dst (ConvF2L (LoadF src)));
14247 format %{ "evcvttss2sisq $dst, $src" %}
14248 ins_encode %{
14249 __ evcvttss2sisq($dst$$Register, $src$$Address);
14250 %}
14251 ins_pipe(pipe_slow);
14252 %}
14253
14254 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
14255 %{
14256 predicate(!VM_Version::supports_avx10_2());
14257 match(Set dst (ConvD2I src));
14258 effect(KILL cr);
14259 format %{ "convert_d2i $dst, $src"%}
14260 ins_encode %{
14261 __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
14262 %}
14263 ins_pipe(pipe_slow);
14264 %}
14265
14266 instruct convD2I_reg_reg_avx10_2(rRegI dst, regD src)
14267 %{
14268 predicate(VM_Version::supports_avx10_2());
14269 match(Set dst (ConvD2I src));
14270 format %{ "evcvttsd2sisl $dst, $src" %}
14271 ins_encode %{
14272 __ evcvttsd2sisl($dst$$Register, $src$$XMMRegister);
14273 %}
14274 ins_pipe(pipe_slow);
14275 %}
14276
14277 instruct convD2I_reg_mem_avx10_2(rRegI dst, memory src)
14278 %{
14279 predicate(VM_Version::supports_avx10_2());
14280 match(Set dst (ConvD2I (LoadD src)));
14281 format %{ "evcvttsd2sisl $dst, $src" %}
14282 ins_encode %{
14283 __ evcvttsd2sisl($dst$$Register, $src$$Address);
14284 %}
14285 ins_pipe(pipe_slow);
14286 %}
14287
14288 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
14289 %{
14290 predicate(!VM_Version::supports_avx10_2());
14291 match(Set dst (ConvD2L src));
14292 effect(KILL cr);
14293 format %{ "convert_d2l $dst, $src"%}
14294 ins_encode %{
14295 __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
14296 %}
14297 ins_pipe(pipe_slow);
14298 %}
14299
14300 instruct convD2L_reg_reg_avx10_2(rRegL dst, regD src)
14301 %{
14302 predicate(VM_Version::supports_avx10_2());
14303 match(Set dst (ConvD2L src));
14304 format %{ "evcvttsd2sisq $dst, $src" %}
14305 ins_encode %{
14306 __ evcvttsd2sisq($dst$$Register, $src$$XMMRegister);
14307 %}
14308 ins_pipe(pipe_slow);
14309 %}
14310
14311 instruct convD2L_reg_mem_avx10_2(rRegL dst, memory src)
14312 %{
14313 predicate(VM_Version::supports_avx10_2());
14314 match(Set dst (ConvD2L (LoadD src)));
14315 format %{ "evcvttsd2sisq $dst, $src" %}
14316 ins_encode %{
14317 __ evcvttsd2sisq($dst$$Register, $src$$Address);
14318 %}
14319 ins_pipe(pipe_slow);
14320 %}
14321
14322 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
14323 %{
14324 match(Set dst (RoundD src));
14325 effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
14326 format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
14327 ins_encode %{
14328 __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
14329 %}
14330 ins_pipe(pipe_slow);
14331 %}
14332
14333 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
14334 %{
14335 match(Set dst (RoundF src));
14336 effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
14337 format %{ "round_float $dst,$src" %}
14338 ins_encode %{
14339 __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
14340 %}
14341 ins_pipe(pipe_slow);
14342 %}
14343
14344 instruct convI2F_reg_reg(vlRegF dst, rRegI src)
14345 %{
14346 predicate(!UseXmmI2F);
14347 match(Set dst (ConvI2F src));
14348
14349 format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
14350 ins_encode %{
14351 if (UseAVX > 0) {
14352 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14353 }
14354 __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
14355 %}
14356 ins_pipe(pipe_slow); // XXX
14357 %}
14358
14359 instruct convI2F_reg_mem(regF dst, memory src)
14360 %{
14361 predicate(UseAVX == 0);
14362 match(Set dst (ConvI2F (LoadI src)));
14363
14364 format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
14365 ins_encode %{
14366 __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
14367 %}
14368 ins_pipe(pipe_slow); // XXX
14369 %}
14370
14371 instruct convI2D_reg_reg(vlRegD dst, rRegI src)
14372 %{
14373 predicate(!UseXmmI2D);
14374 match(Set dst (ConvI2D src));
14375
14376 format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
14377 ins_encode %{
14378 if (UseAVX > 0) {
14379 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14380 }
14381 __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
14382 %}
14383 ins_pipe(pipe_slow); // XXX
14384 %}
14385
14386 instruct convI2D_reg_mem(regD dst, memory src)
14387 %{
14388 predicate(UseAVX == 0);
14389 match(Set dst (ConvI2D (LoadI src)));
14390
14391 format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
14392 ins_encode %{
14393 __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
14394 %}
14395 ins_pipe(pipe_slow); // XXX
14396 %}
14397
14398 instruct convXI2F_reg(regF dst, rRegI src)
14399 %{
14400 predicate(UseXmmI2F);
14401 match(Set dst (ConvI2F src));
14402
14403 format %{ "movdl $dst, $src\n\t"
14404 "cvtdq2psl $dst, $dst\t# i2f" %}
14405 ins_encode %{
14406 __ movdl($dst$$XMMRegister, $src$$Register);
14407 __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
14408 %}
14409 ins_pipe(pipe_slow); // XXX
14410 %}
14411
14412 instruct convXI2D_reg(regD dst, rRegI src)
14413 %{
14414 predicate(UseXmmI2D);
14415 match(Set dst (ConvI2D src));
14416
14417 format %{ "movdl $dst, $src\n\t"
14418 "cvtdq2pdl $dst, $dst\t# i2d" %}
14419 ins_encode %{
14420 __ movdl($dst$$XMMRegister, $src$$Register);
14421 __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
14422 %}
14423 ins_pipe(pipe_slow); // XXX
14424 %}
14425
14426 instruct convL2F_reg_reg(vlRegF dst, rRegL src)
14427 %{
14428 match(Set dst (ConvL2F src));
14429
14430 format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
14431 ins_encode %{
14432 if (UseAVX > 0) {
14433 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14434 }
14435 __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
14436 %}
14437 ins_pipe(pipe_slow); // XXX
14438 %}
14439
14440 instruct convL2F_reg_mem(regF dst, memory src)
14441 %{
14442 predicate(UseAVX == 0);
14443 match(Set dst (ConvL2F (LoadL src)));
14444
14445 format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
14446 ins_encode %{
14447 __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
14448 %}
14449 ins_pipe(pipe_slow); // XXX
14450 %}
14451
14452 instruct convL2D_reg_reg(vlRegD dst, rRegL src)
14453 %{
14454 match(Set dst (ConvL2D src));
14455
14456 format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
14457 ins_encode %{
14458 if (UseAVX > 0) {
14459 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14460 }
14461 __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
14462 %}
14463 ins_pipe(pipe_slow); // XXX
14464 %}
14465
14466 instruct convL2D_reg_mem(regD dst, memory src)
14467 %{
14468 predicate(UseAVX == 0);
14469 match(Set dst (ConvL2D (LoadL src)));
14470
14471 format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
14472 ins_encode %{
14473 __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
14474 %}
14475 ins_pipe(pipe_slow); // XXX
14476 %}
14477
14478 instruct convI2L_reg_reg(rRegL dst, rRegI src)
14479 %{
14480 match(Set dst (ConvI2L src));
14481
14482 ins_cost(125);
14483 format %{ "movslq $dst, $src\t# i2l" %}
14484 ins_encode %{
14485 __ movslq($dst$$Register, $src$$Register);
14486 %}
14487 ins_pipe(ialu_reg_reg);
14488 %}
14489
14490 // Zero-extend convert int to long
14491 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
14492 %{
14493 match(Set dst (AndL (ConvI2L src) mask));
14494
14495 format %{ "movl $dst, $src\t# i2l zero-extend\n\t" %}
14496 ins_encode %{
14497 if ($dst$$reg != $src$$reg) {
14498 __ movl($dst$$Register, $src$$Register);
14499 }
14500 %}
14501 ins_pipe(ialu_reg_reg);
14502 %}
14503
14504 // Zero-extend convert int to long
14505 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
14506 %{
14507 match(Set dst (AndL (ConvI2L (LoadI src)) mask));
14508
14509 format %{ "movl $dst, $src\t# i2l zero-extend\n\t" %}
14510 ins_encode %{
14511 __ movl($dst$$Register, $src$$Address);
14512 %}
14513 ins_pipe(ialu_reg_mem);
14514 %}
14515
14516 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
14517 %{
14518 match(Set dst (AndL src mask));
14519
14520 format %{ "movl $dst, $src\t# zero-extend long" %}
14521 ins_encode %{
14522 __ movl($dst$$Register, $src$$Register);
14523 %}
14524 ins_pipe(ialu_reg_reg);
14525 %}
14526
14527 instruct convL2I_reg_reg(rRegI dst, rRegL src)
14528 %{
14529 match(Set dst (ConvL2I src));
14530
14531 format %{ "movl $dst, $src\t# l2i" %}
14532 ins_encode %{
14533 __ movl($dst$$Register, $src$$Register);
14534 %}
14535 ins_pipe(ialu_reg_reg);
14536 %}
14537
14538
14539 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
14540 match(Set dst (MoveF2I src));
14541 effect(DEF dst, USE src);
14542
14543 ins_cost(125);
14544 format %{ "movl $dst, $src\t# MoveF2I_stack_reg" %}
14545 ins_encode %{
14546 __ movl($dst$$Register, Address(rsp, $src$$disp));
14547 %}
14548 ins_pipe(ialu_reg_mem);
14549 %}
14550
14551 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
14552 match(Set dst (MoveI2F src));
14553 effect(DEF dst, USE src);
14554
14555 ins_cost(125);
14556 format %{ "movss $dst, $src\t# MoveI2F_stack_reg" %}
14557 ins_encode %{
14558 __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
14559 %}
14560 ins_pipe(pipe_slow);
14561 %}
14562
14563 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
14564 match(Set dst (MoveD2L src));
14565 effect(DEF dst, USE src);
14566
14567 ins_cost(125);
14568 format %{ "movq $dst, $src\t# MoveD2L_stack_reg" %}
14569 ins_encode %{
14570 __ movq($dst$$Register, Address(rsp, $src$$disp));
14571 %}
14572 ins_pipe(ialu_reg_mem);
14573 %}
14574
14575 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
14576 predicate(!UseXmmLoadAndClearUpper);
14577 match(Set dst (MoveL2D src));
14578 effect(DEF dst, USE src);
14579
14580 ins_cost(125);
14581 format %{ "movlpd $dst, $src\t# MoveL2D_stack_reg" %}
14582 ins_encode %{
14583 __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
14584 %}
14585 ins_pipe(pipe_slow);
14586 %}
14587
14588 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
14589 predicate(UseXmmLoadAndClearUpper);
14590 match(Set dst (MoveL2D src));
14591 effect(DEF dst, USE src);
14592
14593 ins_cost(125);
14594 format %{ "movsd $dst, $src\t# MoveL2D_stack_reg" %}
14595 ins_encode %{
14596 __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
14597 %}
14598 ins_pipe(pipe_slow);
14599 %}
14600
14601
14602 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
14603 match(Set dst (MoveF2I src));
14604 effect(DEF dst, USE src);
14605
14606 ins_cost(95); // XXX
14607 format %{ "movss $dst, $src\t# MoveF2I_reg_stack" %}
14608 ins_encode %{
14609 __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
14610 %}
14611 ins_pipe(pipe_slow);
14612 %}
14613
14614 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
14615 match(Set dst (MoveI2F src));
14616 effect(DEF dst, USE src);
14617
14618 ins_cost(100);
14619 format %{ "movl $dst, $src\t# MoveI2F_reg_stack" %}
14620 ins_encode %{
14621 __ movl(Address(rsp, $dst$$disp), $src$$Register);
14622 %}
14623 ins_pipe( ialu_mem_reg );
14624 %}
14625
14626 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
14627 match(Set dst (MoveD2L src));
14628 effect(DEF dst, USE src);
14629
14630 ins_cost(95); // XXX
14631 format %{ "movsd $dst, $src\t# MoveL2D_reg_stack" %}
14632 ins_encode %{
14633 __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
14634 %}
14635 ins_pipe(pipe_slow);
14636 %}
14637
14638 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
14639 match(Set dst (MoveL2D src));
14640 effect(DEF dst, USE src);
14641
14642 ins_cost(100);
14643 format %{ "movq $dst, $src\t# MoveL2D_reg_stack" %}
14644 ins_encode %{
14645 __ movq(Address(rsp, $dst$$disp), $src$$Register);
14646 %}
14647 ins_pipe(ialu_mem_reg);
14648 %}
14649
14650 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
14651 match(Set dst (MoveF2I src));
14652 effect(DEF dst, USE src);
14653 ins_cost(85);
14654 format %{ "movd $dst,$src\t# MoveF2I" %}
14655 ins_encode %{
14656 __ movdl($dst$$Register, $src$$XMMRegister);
14657 %}
14658 ins_pipe( pipe_slow );
14659 %}
14660
14661 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
14662 match(Set dst (MoveD2L src));
14663 effect(DEF dst, USE src);
14664 ins_cost(85);
14665 format %{ "movd $dst,$src\t# MoveD2L" %}
14666 ins_encode %{
14667 __ movdq($dst$$Register, $src$$XMMRegister);
14668 %}
14669 ins_pipe( pipe_slow );
14670 %}
14671
14672 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
14673 match(Set dst (MoveI2F src));
14674 effect(DEF dst, USE src);
14675 ins_cost(100);
14676 format %{ "movd $dst,$src\t# MoveI2F" %}
14677 ins_encode %{
14678 __ movdl($dst$$XMMRegister, $src$$Register);
14679 %}
14680 ins_pipe( pipe_slow );
14681 %}
14682
14683 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
14684 match(Set dst (MoveL2D src));
14685 effect(DEF dst, USE src);
14686 ins_cost(100);
14687 format %{ "movd $dst,$src\t# MoveL2D" %}
14688 ins_encode %{
14689 __ movdq($dst$$XMMRegister, $src$$Register);
14690 %}
14691 ins_pipe( pipe_slow );
14692 %}
14693
14694 // Fast clearing of an array
14695 // Small non-constant lenght ClearArray for non-AVX512 targets.
14696 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
14697 Universe dummy, rFlagsReg cr)
14698 %{
14699 predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
14700 match(Set dummy (ClearArray cnt base));
14701 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
14702
14703 format %{ $$template
14704 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14705 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14706 $$emit$$"jg LARGE\n\t"
14707 $$emit$$"dec rcx\n\t"
14708 $$emit$$"js DONE\t# Zero length\n\t"
14709 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14710 $$emit$$"dec rcx\n\t"
14711 $$emit$$"jge LOOP\n\t"
14712 $$emit$$"jmp DONE\n\t"
14713 $$emit$$"# LARGE:\n\t"
14714 if (UseFastStosb) {
14715 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14716 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14717 } else if (UseXMMForObjInit) {
14718 $$emit$$"mov rdi,rax\n\t"
14719 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14720 $$emit$$"jmpq L_zero_64_bytes\n\t"
14721 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14722 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14723 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14724 $$emit$$"add 0x40,rax\n\t"
14725 $$emit$$"# L_zero_64_bytes:\n\t"
14726 $$emit$$"sub 0x8,rcx\n\t"
14727 $$emit$$"jge L_loop\n\t"
14728 $$emit$$"add 0x4,rcx\n\t"
14729 $$emit$$"jl L_tail\n\t"
14730 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14731 $$emit$$"add 0x20,rax\n\t"
14732 $$emit$$"sub 0x4,rcx\n\t"
14733 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14734 $$emit$$"add 0x4,rcx\n\t"
14735 $$emit$$"jle L_end\n\t"
14736 $$emit$$"dec rcx\n\t"
14737 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14738 $$emit$$"vmovq xmm0,(rax)\n\t"
14739 $$emit$$"add 0x8,rax\n\t"
14740 $$emit$$"dec rcx\n\t"
14741 $$emit$$"jge L_sloop\n\t"
14742 $$emit$$"# L_end:\n\t"
14743 } else {
14744 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14745 }
14746 $$emit$$"# DONE"
14747 %}
14748 ins_encode %{
14749 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14750 $tmp$$XMMRegister, false, knoreg);
14751 %}
14752 ins_pipe(pipe_slow);
14753 %}
14754
14755 // Small non-constant length ClearArray for AVX512 targets.
14756 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
14757 Universe dummy, rFlagsReg cr)
14758 %{
14759 predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
14760 match(Set dummy (ClearArray cnt base));
14761 ins_cost(125);
14762 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
14763
14764 format %{ $$template
14765 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14766 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14767 $$emit$$"jg LARGE\n\t"
14768 $$emit$$"dec rcx\n\t"
14769 $$emit$$"js DONE\t# Zero length\n\t"
14770 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14771 $$emit$$"dec rcx\n\t"
14772 $$emit$$"jge LOOP\n\t"
14773 $$emit$$"jmp DONE\n\t"
14774 $$emit$$"# LARGE:\n\t"
14775 if (UseFastStosb) {
14776 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14777 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14778 } else if (UseXMMForObjInit) {
14779 $$emit$$"mov rdi,rax\n\t"
14780 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14781 $$emit$$"jmpq L_zero_64_bytes\n\t"
14782 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14783 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14784 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14785 $$emit$$"add 0x40,rax\n\t"
14786 $$emit$$"# L_zero_64_bytes:\n\t"
14787 $$emit$$"sub 0x8,rcx\n\t"
14788 $$emit$$"jge L_loop\n\t"
14789 $$emit$$"add 0x4,rcx\n\t"
14790 $$emit$$"jl L_tail\n\t"
14791 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14792 $$emit$$"add 0x20,rax\n\t"
14793 $$emit$$"sub 0x4,rcx\n\t"
14794 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14795 $$emit$$"add 0x4,rcx\n\t"
14796 $$emit$$"jle L_end\n\t"
14797 $$emit$$"dec rcx\n\t"
14798 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14799 $$emit$$"vmovq xmm0,(rax)\n\t"
14800 $$emit$$"add 0x8,rax\n\t"
14801 $$emit$$"dec rcx\n\t"
14802 $$emit$$"jge L_sloop\n\t"
14803 $$emit$$"# L_end:\n\t"
14804 } else {
14805 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14806 }
14807 $$emit$$"# DONE"
14808 %}
14809 ins_encode %{
14810 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14811 $tmp$$XMMRegister, false, $ktmp$$KRegister);
14812 %}
14813 ins_pipe(pipe_slow);
14814 %}
14815
14816 // Large non-constant length ClearArray for non-AVX512 targets.
14817 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
14818 Universe dummy, rFlagsReg cr)
14819 %{
14820 predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
14821 match(Set dummy (ClearArray cnt base));
14822 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
14823
14824 format %{ $$template
14825 if (UseFastStosb) {
14826 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14827 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14828 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
14829 } else if (UseXMMForObjInit) {
14830 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
14831 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14832 $$emit$$"jmpq L_zero_64_bytes\n\t"
14833 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14834 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14835 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14836 $$emit$$"add 0x40,rax\n\t"
14837 $$emit$$"# L_zero_64_bytes:\n\t"
14838 $$emit$$"sub 0x8,rcx\n\t"
14839 $$emit$$"jge L_loop\n\t"
14840 $$emit$$"add 0x4,rcx\n\t"
14841 $$emit$$"jl L_tail\n\t"
14842 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14843 $$emit$$"add 0x20,rax\n\t"
14844 $$emit$$"sub 0x4,rcx\n\t"
14845 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14846 $$emit$$"add 0x4,rcx\n\t"
14847 $$emit$$"jle L_end\n\t"
14848 $$emit$$"dec rcx\n\t"
14849 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14850 $$emit$$"vmovq xmm0,(rax)\n\t"
14851 $$emit$$"add 0x8,rax\n\t"
14852 $$emit$$"dec rcx\n\t"
14853 $$emit$$"jge L_sloop\n\t"
14854 $$emit$$"# L_end:\n\t"
14855 } else {
14856 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14857 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
14858 }
14859 %}
14860 ins_encode %{
14861 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14862 $tmp$$XMMRegister, true, knoreg);
14863 %}
14864 ins_pipe(pipe_slow);
14865 %}
14866
14867 // Large non-constant length ClearArray for AVX512 targets.
14868 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
14869 Universe dummy, rFlagsReg cr)
14870 %{
14871 predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
14872 match(Set dummy (ClearArray cnt base));
14873 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
14874
14875 format %{ $$template
14876 if (UseFastStosb) {
14877 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14878 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14879 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
14880 } else if (UseXMMForObjInit) {
14881 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
14882 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14883 $$emit$$"jmpq L_zero_64_bytes\n\t"
14884 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14885 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14886 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14887 $$emit$$"add 0x40,rax\n\t"
14888 $$emit$$"# L_zero_64_bytes:\n\t"
14889 $$emit$$"sub 0x8,rcx\n\t"
14890 $$emit$$"jge L_loop\n\t"
14891 $$emit$$"add 0x4,rcx\n\t"
14892 $$emit$$"jl L_tail\n\t"
14893 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14894 $$emit$$"add 0x20,rax\n\t"
14895 $$emit$$"sub 0x4,rcx\n\t"
14896 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14897 $$emit$$"add 0x4,rcx\n\t"
14898 $$emit$$"jle L_end\n\t"
14899 $$emit$$"dec rcx\n\t"
14900 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14901 $$emit$$"vmovq xmm0,(rax)\n\t"
14902 $$emit$$"add 0x8,rax\n\t"
14903 $$emit$$"dec rcx\n\t"
14904 $$emit$$"jge L_sloop\n\t"
14905 $$emit$$"# L_end:\n\t"
14906 } else {
14907 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14908 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
14909 }
14910 %}
14911 ins_encode %{
14912 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14913 $tmp$$XMMRegister, true, $ktmp$$KRegister);
14914 %}
14915 ins_pipe(pipe_slow);
14916 %}
14917
14918 // Small constant length ClearArray for AVX512 targets.
14919 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
14920 %{
14921 predicate(!((ClearArrayNode*)n)->is_large() && (MaxVectorSize >= 32) && VM_Version::supports_avx512vl());
14922 match(Set dummy (ClearArray cnt base));
14923 ins_cost(100);
14924 effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
14925 format %{ "clear_mem_imm $base , $cnt \n\t" %}
14926 ins_encode %{
14927 __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
14928 %}
14929 ins_pipe(pipe_slow);
14930 %}
14931
14932 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
14933 rax_RegI result, legRegD tmp1, rFlagsReg cr)
14934 %{
14935 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
14936 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14937 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14938
14939 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
14940 ins_encode %{
14941 __ string_compare($str1$$Register, $str2$$Register,
14942 $cnt1$$Register, $cnt2$$Register, $result$$Register,
14943 $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
14944 %}
14945 ins_pipe( pipe_slow );
14946 %}
14947
14948 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
14949 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
14950 %{
14951 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
14952 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14953 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14954
14955 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
14956 ins_encode %{
14957 __ string_compare($str1$$Register, $str2$$Register,
14958 $cnt1$$Register, $cnt2$$Register, $result$$Register,
14959 $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
14960 %}
14961 ins_pipe( pipe_slow );
14962 %}
14963
14964 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
14965 rax_RegI result, legRegD tmp1, rFlagsReg cr)
14966 %{
14967 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
14968 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14969 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14970
14971 format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
14972 ins_encode %{
14973 __ string_compare($str1$$Register, $str2$$Register,
14974 $cnt1$$Register, $cnt2$$Register, $result$$Register,
14975 $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
14976 %}
14977 ins_pipe( pipe_slow );
14978 %}
14979
14980 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
14981 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
14982 %{
14983 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
14984 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14985 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14986
14987 format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
14988 ins_encode %{
14989 __ string_compare($str1$$Register, $str2$$Register,
14990 $cnt1$$Register, $cnt2$$Register, $result$$Register,
14991 $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
14992 %}
14993 ins_pipe( pipe_slow );
14994 %}
14995
14996 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
14997 rax_RegI result, legRegD tmp1, rFlagsReg cr)
14998 %{
14999 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
15000 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15001 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15002
15003 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15004 ins_encode %{
15005 __ string_compare($str1$$Register, $str2$$Register,
15006 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15007 $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
15008 %}
15009 ins_pipe( pipe_slow );
15010 %}
15011
15012 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
15013 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
15014 %{
15015 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
15016 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15017 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15018
15019 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15020 ins_encode %{
15021 __ string_compare($str1$$Register, $str2$$Register,
15022 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15023 $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
15024 %}
15025 ins_pipe( pipe_slow );
15026 %}
15027
15028 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
15029 rax_RegI result, legRegD tmp1, rFlagsReg cr)
15030 %{
15031 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
15032 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15033 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15034
15035 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15036 ins_encode %{
15037 __ string_compare($str2$$Register, $str1$$Register,
15038 $cnt2$$Register, $cnt1$$Register, $result$$Register,
15039 $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
15040 %}
15041 ins_pipe( pipe_slow );
15042 %}
15043
15044 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
15045 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
15046 %{
15047 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
15048 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15049 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15050
15051 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15052 ins_encode %{
15053 __ string_compare($str2$$Register, $str1$$Register,
15054 $cnt2$$Register, $cnt1$$Register, $result$$Register,
15055 $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
15056 %}
15057 ins_pipe( pipe_slow );
15058 %}
15059
15060 // fast search of substring with known size.
15061 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
15062 rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
15063 %{
15064 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
15065 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15066 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
15067
15068 format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
15069 ins_encode %{
15070 int icnt2 = (int)$int_cnt2$$constant;
15071 if (icnt2 >= 16) {
15072 // IndexOf for constant substrings with size >= 16 elements
15073 // which don't need to be loaded through stack.
15074 __ string_indexofC8($str1$$Register, $str2$$Register,
15075 $cnt1$$Register, $cnt2$$Register,
15076 icnt2, $result$$Register,
15077 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
15078 } else {
15079 // Small strings are loaded through stack if they cross page boundary.
15080 __ string_indexof($str1$$Register, $str2$$Register,
15081 $cnt1$$Register, $cnt2$$Register,
15082 icnt2, $result$$Register,
15083 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
15084 }
15085 %}
15086 ins_pipe( pipe_slow );
15087 %}
15088
15089 // fast search of substring with known size.
15090 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
15091 rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
15092 %{
15093 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
15094 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15095 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
15096
15097 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
15098 ins_encode %{
15099 int icnt2 = (int)$int_cnt2$$constant;
15100 if (icnt2 >= 8) {
15101 // IndexOf for constant substrings with size >= 8 elements
15102 // which don't need to be loaded through stack.
15103 __ string_indexofC8($str1$$Register, $str2$$Register,
15104 $cnt1$$Register, $cnt2$$Register,
15105 icnt2, $result$$Register,
15106 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
15107 } else {
15108 // Small strings are loaded through stack if they cross page boundary.
15109 __ string_indexof($str1$$Register, $str2$$Register,
15110 $cnt1$$Register, $cnt2$$Register,
15111 icnt2, $result$$Register,
15112 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
15113 }
15114 %}
15115 ins_pipe( pipe_slow );
15116 %}
15117
15118 // fast search of substring with known size.
15119 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
15120 rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
15121 %{
15122 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
15123 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15124 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
15125
15126 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
15127 ins_encode %{
15128 int icnt2 = (int)$int_cnt2$$constant;
15129 if (icnt2 >= 8) {
15130 // IndexOf for constant substrings with size >= 8 elements
15131 // which don't need to be loaded through stack.
15132 __ string_indexofC8($str1$$Register, $str2$$Register,
15133 $cnt1$$Register, $cnt2$$Register,
15134 icnt2, $result$$Register,
15135 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
15136 } else {
15137 // Small strings are loaded through stack if they cross page boundary.
15138 __ string_indexof($str1$$Register, $str2$$Register,
15139 $cnt1$$Register, $cnt2$$Register,
15140 icnt2, $result$$Register,
15141 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
15142 }
15143 %}
15144 ins_pipe( pipe_slow );
15145 %}
15146
15147 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
15148 rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
15149 %{
15150 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
15151 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15152 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
15153
15154 format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
15155 ins_encode %{
15156 __ string_indexof($str1$$Register, $str2$$Register,
15157 $cnt1$$Register, $cnt2$$Register,
15158 (-1), $result$$Register,
15159 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
15160 %}
15161 ins_pipe( pipe_slow );
15162 %}
15163
15164 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
15165 rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
15166 %{
15167 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
15168 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15169 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
15170
15171 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
15172 ins_encode %{
15173 __ string_indexof($str1$$Register, $str2$$Register,
15174 $cnt1$$Register, $cnt2$$Register,
15175 (-1), $result$$Register,
15176 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
15177 %}
15178 ins_pipe( pipe_slow );
15179 %}
15180
15181 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
15182 rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
15183 %{
15184 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
15185 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15186 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
15187
15188 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
15189 ins_encode %{
15190 __ string_indexof($str1$$Register, $str2$$Register,
15191 $cnt1$$Register, $cnt2$$Register,
15192 (-1), $result$$Register,
15193 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
15194 %}
15195 ins_pipe( pipe_slow );
15196 %}
15197
15198 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
15199 rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
15200 %{
15201 predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15202 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15203 effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
15204 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result // KILL all" %}
15205 ins_encode %{
15206 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
15207 $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
15208 %}
15209 ins_pipe( pipe_slow );
15210 %}
15211
15212 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
15213 rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
15214 %{
15215 predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15216 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15217 effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
15218 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result // KILL all" %}
15219 ins_encode %{
15220 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
15221 $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
15222 %}
15223 ins_pipe( pipe_slow );
15224 %}
15225
15226 // fast string equals
15227 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
15228 legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
15229 %{
15230 predicate(!VM_Version::supports_avx512vlbw());
15231 match(Set result (StrEquals (Binary str1 str2) cnt));
15232 effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
15233
15234 format %{ "String Equals $str1,$str2,$cnt -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15235 ins_encode %{
15236 __ arrays_equals(false, $str1$$Register, $str2$$Register,
15237 $cnt$$Register, $result$$Register, $tmp3$$Register,
15238 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
15239 %}
15240 ins_pipe( pipe_slow );
15241 %}
15242
15243 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
15244 legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
15245 %{
15246 predicate(VM_Version::supports_avx512vlbw());
15247 match(Set result (StrEquals (Binary str1 str2) cnt));
15248 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
15249
15250 format %{ "String Equals $str1,$str2,$cnt -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15251 ins_encode %{
15252 __ arrays_equals(false, $str1$$Register, $str2$$Register,
15253 $cnt$$Register, $result$$Register, $tmp3$$Register,
15254 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
15255 %}
15256 ins_pipe( pipe_slow );
15257 %}
15258
15259 // fast array equals
15260 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15261 legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15262 %{
15263 predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
15264 match(Set result (AryEq ary1 ary2));
15265 effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15266
15267 format %{ "Array Equals byte[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15268 ins_encode %{
15269 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15270 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15271 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
15272 %}
15273 ins_pipe( pipe_slow );
15274 %}
15275
15276 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15277 legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15278 %{
15279 predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
15280 match(Set result (AryEq ary1 ary2));
15281 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15282
15283 format %{ "Array Equals byte[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15284 ins_encode %{
15285 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15286 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15287 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
15288 %}
15289 ins_pipe( pipe_slow );
15290 %}
15291
15292 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15293 legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15294 %{
15295 predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
15296 match(Set result (AryEq ary1 ary2));
15297 effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15298
15299 format %{ "Array Equals char[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15300 ins_encode %{
15301 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15302 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15303 $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
15304 %}
15305 ins_pipe( pipe_slow );
15306 %}
15307
15308 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15309 legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15310 %{
15311 predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
15312 match(Set result (AryEq ary1 ary2));
15313 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15314
15315 format %{ "Array Equals char[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15316 ins_encode %{
15317 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15318 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15319 $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
15320 %}
15321 ins_pipe( pipe_slow );
15322 %}
15323
15324 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
15325 legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
15326 legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
15327 legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
15328 legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
15329 %{
15330 predicate(UseAVX >= 2);
15331 match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
15332 effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
15333 TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
15334 TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
15335 USE basic_type, KILL cr);
15336
15337 format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result // KILL all" %}
15338 ins_encode %{
15339 __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
15340 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
15341 $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
15342 $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
15343 $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
15344 $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
15345 $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
15346 %}
15347 ins_pipe( pipe_slow );
15348 %}
15349
15350 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
15351 legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
15352 %{
15353 predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
15354 match(Set result (CountPositives ary1 len));
15355 effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
15356
15357 format %{ "countPositives byte[] $ary1,$len -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15358 ins_encode %{
15359 __ count_positives($ary1$$Register, $len$$Register,
15360 $result$$Register, $tmp3$$Register,
15361 $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
15362 %}
15363 ins_pipe( pipe_slow );
15364 %}
15365
15366 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
15367 legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
15368 %{
15369 predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
15370 match(Set result (CountPositives ary1 len));
15371 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
15372
15373 format %{ "countPositives byte[] $ary1,$len -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15374 ins_encode %{
15375 __ count_positives($ary1$$Register, $len$$Register,
15376 $result$$Register, $tmp3$$Register,
15377 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
15378 %}
15379 ins_pipe( pipe_slow );
15380 %}
15381
15382 // fast char[] to byte[] compression
15383 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
15384 legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15385 predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
15386 match(Set result (StrCompressedCopy src (Binary dst len)));
15387 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
15388 USE_KILL len, KILL tmp5, KILL cr);
15389
15390 format %{ "String Compress $src,$dst -> $result // KILL RAX, RCX, RDX" %}
15391 ins_encode %{
15392 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
15393 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15394 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
15395 knoreg, knoreg);
15396 %}
15397 ins_pipe( pipe_slow );
15398 %}
15399
15400 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
15401 legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15402 predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
15403 match(Set result (StrCompressedCopy src (Binary dst len)));
15404 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
15405 USE_KILL len, KILL tmp5, KILL cr);
15406
15407 format %{ "String Compress $src,$dst -> $result // KILL RAX, RCX, RDX" %}
15408 ins_encode %{
15409 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
15410 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15411 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
15412 $ktmp1$$KRegister, $ktmp2$$KRegister);
15413 %}
15414 ins_pipe( pipe_slow );
15415 %}
15416 // fast byte[] to char[] inflation
15417 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15418 legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
15419 predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
15420 match(Set dummy (StrInflatedCopy src (Binary dst len)));
15421 effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
15422
15423 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
15424 ins_encode %{
15425 __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
15426 $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
15427 %}
15428 ins_pipe( pipe_slow );
15429 %}
15430
15431 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15432 legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
15433 predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
15434 match(Set dummy (StrInflatedCopy src (Binary dst len)));
15435 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
15436
15437 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
15438 ins_encode %{
15439 __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
15440 $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
15441 %}
15442 ins_pipe( pipe_slow );
15443 %}
15444
15445 // encode char[] to byte[] in ISO_8859_1
15446 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15447 legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
15448 rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15449 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
15450 match(Set result (EncodeISOArray src (Binary dst len)));
15451 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
15452
15453 format %{ "Encode iso array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
15454 ins_encode %{
15455 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
15456 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15457 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
15458 %}
15459 ins_pipe( pipe_slow );
15460 %}
15461
15462 // encode char[] to byte[] in ASCII
15463 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15464 legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
15465 rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15466 predicate(((EncodeISOArrayNode*)n)->is_ascii());
15467 match(Set result (EncodeISOArray src (Binary dst len)));
15468 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
15469
15470 format %{ "Encode ascii array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
15471 ins_encode %{
15472 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
15473 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15474 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
15475 %}
15476 ins_pipe( pipe_slow );
15477 %}
15478
15479 //----------Overflow Math Instructions-----------------------------------------
15480
15481 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
15482 %{
15483 match(Set cr (OverflowAddI op1 op2));
15484 effect(DEF cr, USE_KILL op1, USE op2);
15485
15486 format %{ "addl $op1, $op2\t# overflow check int" %}
15487
15488 ins_encode %{
15489 __ addl($op1$$Register, $op2$$Register);
15490 %}
15491 ins_pipe(ialu_reg_reg);
15492 %}
15493
15494 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
15495 %{
15496 match(Set cr (OverflowAddI op1 op2));
15497 effect(DEF cr, USE_KILL op1, USE op2);
15498
15499 format %{ "addl $op1, $op2\t# overflow check int" %}
15500
15501 ins_encode %{
15502 __ addl($op1$$Register, $op2$$constant);
15503 %}
15504 ins_pipe(ialu_reg_reg);
15505 %}
15506
15507 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
15508 %{
15509 match(Set cr (OverflowAddL op1 op2));
15510 effect(DEF cr, USE_KILL op1, USE op2);
15511
15512 format %{ "addq $op1, $op2\t# overflow check long" %}
15513 ins_encode %{
15514 __ addq($op1$$Register, $op2$$Register);
15515 %}
15516 ins_pipe(ialu_reg_reg);
15517 %}
15518
15519 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
15520 %{
15521 match(Set cr (OverflowAddL op1 op2));
15522 effect(DEF cr, USE_KILL op1, USE op2);
15523
15524 format %{ "addq $op1, $op2\t# overflow check long" %}
15525 ins_encode %{
15526 __ addq($op1$$Register, $op2$$constant);
15527 %}
15528 ins_pipe(ialu_reg_reg);
15529 %}
15530
15531 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
15532 %{
15533 match(Set cr (OverflowSubI op1 op2));
15534
15535 format %{ "cmpl $op1, $op2\t# overflow check int" %}
15536 ins_encode %{
15537 __ cmpl($op1$$Register, $op2$$Register);
15538 %}
15539 ins_pipe(ialu_reg_reg);
15540 %}
15541
15542 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
15543 %{
15544 match(Set cr (OverflowSubI op1 op2));
15545
15546 format %{ "cmpl $op1, $op2\t# overflow check int" %}
15547 ins_encode %{
15548 __ cmpl($op1$$Register, $op2$$constant);
15549 %}
15550 ins_pipe(ialu_reg_reg);
15551 %}
15552
15553 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
15554 %{
15555 match(Set cr (OverflowSubL op1 op2));
15556
15557 format %{ "cmpq $op1, $op2\t# overflow check long" %}
15558 ins_encode %{
15559 __ cmpq($op1$$Register, $op2$$Register);
15560 %}
15561 ins_pipe(ialu_reg_reg);
15562 %}
15563
15564 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
15565 %{
15566 match(Set cr (OverflowSubL op1 op2));
15567
15568 format %{ "cmpq $op1, $op2\t# overflow check long" %}
15569 ins_encode %{
15570 __ cmpq($op1$$Register, $op2$$constant);
15571 %}
15572 ins_pipe(ialu_reg_reg);
15573 %}
15574
15575 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
15576 %{
15577 match(Set cr (OverflowSubI zero op2));
15578 effect(DEF cr, USE_KILL op2);
15579
15580 format %{ "negl $op2\t# overflow check int" %}
15581 ins_encode %{
15582 __ negl($op2$$Register);
15583 %}
15584 ins_pipe(ialu_reg_reg);
15585 %}
15586
15587 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
15588 %{
15589 match(Set cr (OverflowSubL zero op2));
15590 effect(DEF cr, USE_KILL op2);
15591
15592 format %{ "negq $op2\t# overflow check long" %}
15593 ins_encode %{
15594 __ negq($op2$$Register);
15595 %}
15596 ins_pipe(ialu_reg_reg);
15597 %}
15598
15599 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
15600 %{
15601 match(Set cr (OverflowMulI op1 op2));
15602 effect(DEF cr, USE_KILL op1, USE op2);
15603
15604 format %{ "imull $op1, $op2\t# overflow check int" %}
15605 ins_encode %{
15606 __ imull($op1$$Register, $op2$$Register);
15607 %}
15608 ins_pipe(ialu_reg_reg_alu0);
15609 %}
15610
15611 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
15612 %{
15613 match(Set cr (OverflowMulI op1 op2));
15614 effect(DEF cr, TEMP tmp, USE op1, USE op2);
15615
15616 format %{ "imull $tmp, $op1, $op2\t# overflow check int" %}
15617 ins_encode %{
15618 __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
15619 %}
15620 ins_pipe(ialu_reg_reg_alu0);
15621 %}
15622
15623 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
15624 %{
15625 match(Set cr (OverflowMulL op1 op2));
15626 effect(DEF cr, USE_KILL op1, USE op2);
15627
15628 format %{ "imulq $op1, $op2\t# overflow check long" %}
15629 ins_encode %{
15630 __ imulq($op1$$Register, $op2$$Register);
15631 %}
15632 ins_pipe(ialu_reg_reg_alu0);
15633 %}
15634
15635 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
15636 %{
15637 match(Set cr (OverflowMulL op1 op2));
15638 effect(DEF cr, TEMP tmp, USE op1, USE op2);
15639
15640 format %{ "imulq $tmp, $op1, $op2\t# overflow check long" %}
15641 ins_encode %{
15642 __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
15643 %}
15644 ins_pipe(ialu_reg_reg_alu0);
15645 %}
15646
15647
15648 //----------Control Flow Instructions------------------------------------------
15649 // Signed compare Instructions
15650
15651 // XXX more variants!!
15652 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
15653 %{
15654 match(Set cr (CmpI op1 op2));
15655 effect(DEF cr, USE op1, USE op2);
15656
15657 format %{ "cmpl $op1, $op2" %}
15658 ins_encode %{
15659 __ cmpl($op1$$Register, $op2$$Register);
15660 %}
15661 ins_pipe(ialu_cr_reg_reg);
15662 %}
15663
15664 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
15665 %{
15666 match(Set cr (CmpI op1 op2));
15667
15668 format %{ "cmpl $op1, $op2" %}
15669 ins_encode %{
15670 __ cmpl($op1$$Register, $op2$$constant);
15671 %}
15672 ins_pipe(ialu_cr_reg_imm);
15673 %}
15674
15675 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
15676 %{
15677 match(Set cr (CmpI op1 (LoadI op2)));
15678
15679 ins_cost(500); // XXX
15680 format %{ "cmpl $op1, $op2" %}
15681 ins_encode %{
15682 __ cmpl($op1$$Register, $op2$$Address);
15683 %}
15684 ins_pipe(ialu_cr_reg_mem);
15685 %}
15686
15687 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
15688 %{
15689 match(Set cr (CmpI src zero));
15690
15691 format %{ "testl $src, $src" %}
15692 ins_encode %{
15693 __ testl($src$$Register, $src$$Register);
15694 %}
15695 ins_pipe(ialu_cr_reg_imm);
15696 %}
15697
15698 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
15699 %{
15700 match(Set cr (CmpI (AndI src con) zero));
15701
15702 format %{ "testl $src, $con" %}
15703 ins_encode %{
15704 __ testl($src$$Register, $con$$constant);
15705 %}
15706 ins_pipe(ialu_cr_reg_imm);
15707 %}
15708
15709 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
15710 %{
15711 match(Set cr (CmpI (AndI src1 src2) zero));
15712
15713 format %{ "testl $src1, $src2" %}
15714 ins_encode %{
15715 __ testl($src1$$Register, $src2$$Register);
15716 %}
15717 ins_pipe(ialu_cr_reg_imm);
15718 %}
15719
15720 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
15721 %{
15722 match(Set cr (CmpI (AndI src (LoadI mem)) zero));
15723
15724 format %{ "testl $src, $mem" %}
15725 ins_encode %{
15726 __ testl($src$$Register, $mem$$Address);
15727 %}
15728 ins_pipe(ialu_cr_reg_mem);
15729 %}
15730
15731 // Unsigned compare Instructions; really, same as signed except they
15732 // produce an rFlagsRegU instead of rFlagsReg.
15733 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
15734 %{
15735 match(Set cr (CmpU op1 op2));
15736
15737 format %{ "cmpl $op1, $op2\t# unsigned" %}
15738 ins_encode %{
15739 __ cmpl($op1$$Register, $op2$$Register);
15740 %}
15741 ins_pipe(ialu_cr_reg_reg);
15742 %}
15743
15744 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
15745 %{
15746 match(Set cr (CmpU op1 op2));
15747
15748 format %{ "cmpl $op1, $op2\t# unsigned" %}
15749 ins_encode %{
15750 __ cmpl($op1$$Register, $op2$$constant);
15751 %}
15752 ins_pipe(ialu_cr_reg_imm);
15753 %}
15754
15755 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
15756 %{
15757 match(Set cr (CmpU op1 (LoadI op2)));
15758
15759 ins_cost(500); // XXX
15760 format %{ "cmpl $op1, $op2\t# unsigned" %}
15761 ins_encode %{
15762 __ cmpl($op1$$Register, $op2$$Address);
15763 %}
15764 ins_pipe(ialu_cr_reg_mem);
15765 %}
15766
15767 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
15768 %{
15769 match(Set cr (CmpU src zero));
15770
15771 format %{ "testl $src, $src\t# unsigned" %}
15772 ins_encode %{
15773 __ testl($src$$Register, $src$$Register);
15774 %}
15775 ins_pipe(ialu_cr_reg_imm);
15776 %}
15777
15778 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
15779 %{
15780 match(Set cr (CmpP op1 op2));
15781
15782 format %{ "cmpq $op1, $op2\t# ptr" %}
15783 ins_encode %{
15784 __ cmpq($op1$$Register, $op2$$Register);
15785 %}
15786 ins_pipe(ialu_cr_reg_reg);
15787 %}
15788
15789 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
15790 %{
15791 match(Set cr (CmpP op1 (LoadP op2)));
15792 predicate(n->in(2)->as_Load()->barrier_data() == 0);
15793
15794 ins_cost(500); // XXX
15795 format %{ "cmpq $op1, $op2\t# ptr" %}
15796 ins_encode %{
15797 __ cmpq($op1$$Register, $op2$$Address);
15798 %}
15799 ins_pipe(ialu_cr_reg_mem);
15800 %}
15801
15802 // XXX this is generalized by compP_rReg_mem???
15803 // Compare raw pointer (used in out-of-heap check).
15804 // Only works because non-oop pointers must be raw pointers
15805 // and raw pointers have no anti-dependencies.
15806 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
15807 %{
15808 predicate(n->in(2)->in(2)->bottom_type()->isa_rawptr() != nullptr &&
15809 n->in(2)->as_Load()->barrier_data() == 0);
15810 match(Set cr (CmpP op1 (LoadP op2)));
15811
15812 format %{ "cmpq $op1, $op2\t# raw ptr" %}
15813 ins_encode %{
15814 __ cmpq($op1$$Register, $op2$$Address);
15815 %}
15816 ins_pipe(ialu_cr_reg_mem);
15817 %}
15818
15819 // This will generate a signed flags result. This should be OK since
15820 // any compare to a zero should be eq/neq.
15821 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
15822 %{
15823 match(Set cr (CmpP src zero));
15824
15825 format %{ "testq $src, $src\t# ptr" %}
15826 ins_encode %{
15827 __ testq($src$$Register, $src$$Register);
15828 %}
15829 ins_pipe(ialu_cr_reg_imm);
15830 %}
15831
15832 // This will generate a signed flags result. This should be OK since
15833 // any compare to a zero should be eq/neq.
15834 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
15835 %{
15836 predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
15837 n->in(1)->as_Load()->barrier_data() == 0);
15838 match(Set cr (CmpP (LoadP op) zero));
15839
15840 ins_cost(500); // XXX
15841 format %{ "testq $op, 0xffffffffffffffff\t# ptr" %}
15842 ins_encode %{
15843 __ testq($op$$Address, 0xFFFFFFFF);
15844 %}
15845 ins_pipe(ialu_cr_reg_imm);
15846 %}
15847
15848 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
15849 %{
15850 predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
15851 n->in(1)->as_Load()->barrier_data() == 0);
15852 match(Set cr (CmpP (LoadP mem) zero));
15853
15854 format %{ "cmpq R12, $mem\t# ptr (R12_heapbase==0)" %}
15855 ins_encode %{
15856 __ cmpq(r12, $mem$$Address);
15857 %}
15858 ins_pipe(ialu_cr_reg_mem);
15859 %}
15860
15861 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
15862 %{
15863 match(Set cr (CmpN op1 op2));
15864
15865 format %{ "cmpl $op1, $op2\t# compressed ptr" %}
15866 ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
15867 ins_pipe(ialu_cr_reg_reg);
15868 %}
15869
15870 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
15871 %{
15872 predicate(n->in(2)->as_Load()->barrier_data() == 0);
15873 match(Set cr (CmpN src (LoadN mem)));
15874
15875 format %{ "cmpl $src, $mem\t# compressed ptr" %}
15876 ins_encode %{
15877 __ cmpl($src$$Register, $mem$$Address);
15878 %}
15879 ins_pipe(ialu_cr_reg_mem);
15880 %}
15881
15882 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
15883 match(Set cr (CmpN op1 op2));
15884
15885 format %{ "cmpl $op1, $op2\t# compressed ptr" %}
15886 ins_encode %{
15887 __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
15888 %}
15889 ins_pipe(ialu_cr_reg_imm);
15890 %}
15891
15892 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
15893 %{
15894 predicate(n->in(2)->as_Load()->barrier_data() == 0);
15895 match(Set cr (CmpN src (LoadN mem)));
15896
15897 format %{ "cmpl $mem, $src\t# compressed ptr" %}
15898 ins_encode %{
15899 __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
15900 %}
15901 ins_pipe(ialu_cr_reg_mem);
15902 %}
15903
15904 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
15905 match(Set cr (CmpN op1 op2));
15906
15907 format %{ "cmpl $op1, $op2\t# compressed klass ptr" %}
15908 ins_encode %{
15909 __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
15910 %}
15911 ins_pipe(ialu_cr_reg_imm);
15912 %}
15913
15914 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
15915 %{
15916 predicate(!UseCompactObjectHeaders);
15917 match(Set cr (CmpN src (LoadNKlass mem)));
15918
15919 format %{ "cmpl $mem, $src\t# compressed klass ptr" %}
15920 ins_encode %{
15921 __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
15922 %}
15923 ins_pipe(ialu_cr_reg_mem);
15924 %}
15925
15926 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
15927 match(Set cr (CmpN src zero));
15928
15929 format %{ "testl $src, $src\t# compressed ptr" %}
15930 ins_encode %{ __ testl($src$$Register, $src$$Register); %}
15931 ins_pipe(ialu_cr_reg_imm);
15932 %}
15933
15934 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
15935 %{
15936 predicate(CompressedOops::base() != nullptr &&
15937 n->in(1)->as_Load()->barrier_data() == 0);
15938 match(Set cr (CmpN (LoadN mem) zero));
15939
15940 ins_cost(500); // XXX
15941 format %{ "testl $mem, 0xffffffff\t# compressed ptr" %}
15942 ins_encode %{
15943 __ cmpl($mem$$Address, (int)0xFFFFFFFF);
15944 %}
15945 ins_pipe(ialu_cr_reg_mem);
15946 %}
15947
15948 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
15949 %{
15950 predicate(CompressedOops::base() == nullptr &&
15951 n->in(1)->as_Load()->barrier_data() == 0);
15952 match(Set cr (CmpN (LoadN mem) zero));
15953
15954 format %{ "cmpl R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
15955 ins_encode %{
15956 __ cmpl(r12, $mem$$Address);
15957 %}
15958 ins_pipe(ialu_cr_reg_mem);
15959 %}
15960
15961 // Yanked all unsigned pointer compare operations.
15962 // Pointer compares are done with CmpP which is already unsigned.
15963
15964 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
15965 %{
15966 match(Set cr (CmpL op1 op2));
15967
15968 format %{ "cmpq $op1, $op2" %}
15969 ins_encode %{
15970 __ cmpq($op1$$Register, $op2$$Register);
15971 %}
15972 ins_pipe(ialu_cr_reg_reg);
15973 %}
15974
15975 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
15976 %{
15977 match(Set cr (CmpL op1 op2));
15978
15979 format %{ "cmpq $op1, $op2" %}
15980 ins_encode %{
15981 __ cmpq($op1$$Register, $op2$$constant);
15982 %}
15983 ins_pipe(ialu_cr_reg_imm);
15984 %}
15985
15986 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
15987 %{
15988 match(Set cr (CmpL op1 (LoadL op2)));
15989
15990 format %{ "cmpq $op1, $op2" %}
15991 ins_encode %{
15992 __ cmpq($op1$$Register, $op2$$Address);
15993 %}
15994 ins_pipe(ialu_cr_reg_mem);
15995 %}
15996
15997 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
15998 %{
15999 match(Set cr (CmpL src zero));
16000
16001 format %{ "testq $src, $src" %}
16002 ins_encode %{
16003 __ testq($src$$Register, $src$$Register);
16004 %}
16005 ins_pipe(ialu_cr_reg_imm);
16006 %}
16007
16008 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
16009 %{
16010 match(Set cr (CmpL (AndL src con) zero));
16011
16012 format %{ "testq $src, $con\t# long" %}
16013 ins_encode %{
16014 __ testq($src$$Register, $con$$constant);
16015 %}
16016 ins_pipe(ialu_cr_reg_imm);
16017 %}
16018
16019 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
16020 %{
16021 match(Set cr (CmpL (AndL src1 src2) zero));
16022
16023 format %{ "testq $src1, $src2\t# long" %}
16024 ins_encode %{
16025 __ testq($src1$$Register, $src2$$Register);
16026 %}
16027 ins_pipe(ialu_cr_reg_imm);
16028 %}
16029
16030 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
16031 %{
16032 match(Set cr (CmpL (AndL src (LoadL mem)) zero));
16033
16034 format %{ "testq $src, $mem" %}
16035 ins_encode %{
16036 __ testq($src$$Register, $mem$$Address);
16037 %}
16038 ins_pipe(ialu_cr_reg_mem);
16039 %}
16040
16041 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
16042 %{
16043 match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
16044
16045 format %{ "testq $src, $mem" %}
16046 ins_encode %{
16047 __ testq($src$$Register, $mem$$Address);
16048 %}
16049 ins_pipe(ialu_cr_reg_mem);
16050 %}
16051
16052 // Manifest a CmpU result in an integer register. Very painful.
16053 // This is the test to avoid.
16054 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
16055 %{
16056 match(Set dst (CmpU3 src1 src2));
16057 effect(KILL flags);
16058
16059 ins_cost(275); // XXX
16060 format %{ "cmpl $src1, $src2\t# CmpL3\n\t"
16061 "movl $dst, -1\n\t"
16062 "jb,u done\n\t"
16063 "setcc $dst \t# emits setne + movzbl or setzune for APX"
16064 "done:" %}
16065 ins_encode %{
16066 Label done;
16067 __ cmpl($src1$$Register, $src2$$Register);
16068 __ movl($dst$$Register, -1);
16069 __ jccb(Assembler::below, done);
16070 __ setcc(Assembler::notZero, $dst$$Register);
16071 __ bind(done);
16072 %}
16073 ins_pipe(pipe_slow);
16074 %}
16075
16076 // Manifest a CmpL result in an integer register. Very painful.
16077 // This is the test to avoid.
16078 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
16079 %{
16080 match(Set dst (CmpL3 src1 src2));
16081 effect(KILL flags);
16082
16083 ins_cost(275); // XXX
16084 format %{ "cmpq $src1, $src2\t# CmpL3\n\t"
16085 "movl $dst, -1\n\t"
16086 "jl,s done\n\t"
16087 "setcc $dst \t# emits setne + movzbl or setzune for APX"
16088 "done:" %}
16089 ins_encode %{
16090 Label done;
16091 __ cmpq($src1$$Register, $src2$$Register);
16092 __ movl($dst$$Register, -1);
16093 __ jccb(Assembler::less, done);
16094 __ setcc(Assembler::notZero, $dst$$Register);
16095 __ bind(done);
16096 %}
16097 ins_pipe(pipe_slow);
16098 %}
16099
16100 // Manifest a CmpUL result in an integer register. Very painful.
16101 // This is the test to avoid.
16102 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
16103 %{
16104 match(Set dst (CmpUL3 src1 src2));
16105 effect(KILL flags);
16106
16107 ins_cost(275); // XXX
16108 format %{ "cmpq $src1, $src2\t# CmpL3\n\t"
16109 "movl $dst, -1\n\t"
16110 "jb,u done\n\t"
16111 "setcc $dst \t# emits setne + movzbl or setzune for APX"
16112 "done:" %}
16113 ins_encode %{
16114 Label done;
16115 __ cmpq($src1$$Register, $src2$$Register);
16116 __ movl($dst$$Register, -1);
16117 __ jccb(Assembler::below, done);
16118 __ setcc(Assembler::notZero, $dst$$Register);
16119 __ bind(done);
16120 %}
16121 ins_pipe(pipe_slow);
16122 %}
16123
16124 // Unsigned long compare Instructions; really, same as signed long except they
16125 // produce an rFlagsRegU instead of rFlagsReg.
16126 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
16127 %{
16128 match(Set cr (CmpUL op1 op2));
16129
16130 format %{ "cmpq $op1, $op2\t# unsigned" %}
16131 ins_encode %{
16132 __ cmpq($op1$$Register, $op2$$Register);
16133 %}
16134 ins_pipe(ialu_cr_reg_reg);
16135 %}
16136
16137 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
16138 %{
16139 match(Set cr (CmpUL op1 op2));
16140
16141 format %{ "cmpq $op1, $op2\t# unsigned" %}
16142 ins_encode %{
16143 __ cmpq($op1$$Register, $op2$$constant);
16144 %}
16145 ins_pipe(ialu_cr_reg_imm);
16146 %}
16147
16148 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
16149 %{
16150 match(Set cr (CmpUL op1 (LoadL op2)));
16151
16152 format %{ "cmpq $op1, $op2\t# unsigned" %}
16153 ins_encode %{
16154 __ cmpq($op1$$Register, $op2$$Address);
16155 %}
16156 ins_pipe(ialu_cr_reg_mem);
16157 %}
16158
16159 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
16160 %{
16161 match(Set cr (CmpUL src zero));
16162
16163 format %{ "testq $src, $src\t# unsigned" %}
16164 ins_encode %{
16165 __ testq($src$$Register, $src$$Register);
16166 %}
16167 ins_pipe(ialu_cr_reg_imm);
16168 %}
16169
16170 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
16171 %{
16172 match(Set cr (CmpI (LoadB mem) imm));
16173
16174 ins_cost(125);
16175 format %{ "cmpb $mem, $imm" %}
16176 ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
16177 ins_pipe(ialu_cr_reg_mem);
16178 %}
16179
16180 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
16181 %{
16182 match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
16183
16184 ins_cost(125);
16185 format %{ "testb $mem, $imm\t# ubyte" %}
16186 ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
16187 ins_pipe(ialu_cr_reg_mem);
16188 %}
16189
16190 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
16191 %{
16192 match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
16193
16194 ins_cost(125);
16195 format %{ "testb $mem, $imm\t# byte" %}
16196 ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
16197 ins_pipe(ialu_cr_reg_mem);
16198 %}
16199
16200 //----------Max and Min--------------------------------------------------------
16201 // Min Instructions
16202
16203 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
16204 %{
16205 predicate(!UseAPX);
16206 effect(USE_DEF dst, USE src, USE cr);
16207
16208 format %{ "cmovlgt $dst, $src\t# min" %}
16209 ins_encode %{
16210 __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
16211 %}
16212 ins_pipe(pipe_cmov_reg);
16213 %}
16214
16215 instruct cmovI_reg_g_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
16216 %{
16217 predicate(UseAPX);
16218 effect(DEF dst, USE src1, USE src2, USE cr);
16219
16220 format %{ "ecmovlgt $dst, $src1, $src2\t# min ndd" %}
16221 ins_encode %{
16222 __ ecmovl(Assembler::greater, $dst$$Register, $src1$$Register, $src2$$Register);
16223 %}
16224 ins_pipe(pipe_cmov_reg);
16225 %}
16226
16227 instruct minI_rReg(rRegI dst, rRegI src)
16228 %{
16229 predicate(!UseAPX);
16230 match(Set dst (MinI dst src));
16231
16232 ins_cost(200);
16233 expand %{
16234 rFlagsReg cr;
16235 compI_rReg(cr, dst, src);
16236 cmovI_reg_g(dst, src, cr);
16237 %}
16238 %}
16239
16240 instruct minI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
16241 %{
16242 predicate(UseAPX);
16243 match(Set dst (MinI src1 src2));
16244 effect(DEF dst, USE src1, USE src2);
16245 flag(PD::Flag_ndd_demotable_opr1);
16246
16247 ins_cost(200);
16248 expand %{
16249 rFlagsReg cr;
16250 compI_rReg(cr, src1, src2);
16251 cmovI_reg_g_ndd(dst, src1, src2, cr);
16252 %}
16253 %}
16254
16255 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
16256 %{
16257 predicate(!UseAPX);
16258 effect(USE_DEF dst, USE src, USE cr);
16259
16260 format %{ "cmovllt $dst, $src\t# max" %}
16261 ins_encode %{
16262 __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
16263 %}
16264 ins_pipe(pipe_cmov_reg);
16265 %}
16266
16267 instruct cmovI_reg_l_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
16268 %{
16269 predicate(UseAPX);
16270 effect(DEF dst, USE src1, USE src2, USE cr);
16271
16272 format %{ "ecmovllt $dst, $src1, $src2\t# max ndd" %}
16273 ins_encode %{
16274 __ ecmovl(Assembler::less, $dst$$Register, $src1$$Register, $src2$$Register);
16275 %}
16276 ins_pipe(pipe_cmov_reg);
16277 %}
16278
16279 instruct maxI_rReg(rRegI dst, rRegI src)
16280 %{
16281 predicate(!UseAPX);
16282 match(Set dst (MaxI dst src));
16283
16284 ins_cost(200);
16285 expand %{
16286 rFlagsReg cr;
16287 compI_rReg(cr, dst, src);
16288 cmovI_reg_l(dst, src, cr);
16289 %}
16290 %}
16291
16292 instruct maxI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
16293 %{
16294 predicate(UseAPX);
16295 match(Set dst (MaxI src1 src2));
16296 effect(DEF dst, USE src1, USE src2);
16297 flag(PD::Flag_ndd_demotable_opr1);
16298
16299 ins_cost(200);
16300 expand %{
16301 rFlagsReg cr;
16302 compI_rReg(cr, src1, src2);
16303 cmovI_reg_l_ndd(dst, src1, src2, cr);
16304 %}
16305 %}
16306
16307 // ============================================================================
16308 // Branch Instructions
16309
16310 // Jump Direct - Label defines a relative address from JMP+1
16311 instruct jmpDir(label labl)
16312 %{
16313 match(Goto);
16314 effect(USE labl);
16315
16316 ins_cost(300);
16317 format %{ "jmp $labl" %}
16318 size(5);
16319 ins_encode %{
16320 Label* L = $labl$$label;
16321 __ jmp(*L, false); // Always long jump
16322 %}
16323 ins_pipe(pipe_jmp);
16324 %}
16325
16326 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16327 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
16328 %{
16329 match(If cop cr);
16330 effect(USE labl);
16331
16332 ins_cost(300);
16333 format %{ "j$cop $labl" %}
16334 size(6);
16335 ins_encode %{
16336 Label* L = $labl$$label;
16337 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16338 %}
16339 ins_pipe(pipe_jcc);
16340 %}
16341
16342 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16343 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
16344 %{
16345 match(CountedLoopEnd cop cr);
16346 effect(USE labl);
16347
16348 ins_cost(300);
16349 format %{ "j$cop $labl\t# loop end" %}
16350 size(6);
16351 ins_encode %{
16352 Label* L = $labl$$label;
16353 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16354 %}
16355 ins_pipe(pipe_jcc);
16356 %}
16357
16358 // Jump Direct Conditional - using unsigned comparison
16359 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
16360 match(If cop cmp);
16361 effect(USE labl);
16362
16363 ins_cost(300);
16364 format %{ "j$cop,u $labl" %}
16365 size(6);
16366 ins_encode %{
16367 Label* L = $labl$$label;
16368 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16369 %}
16370 ins_pipe(pipe_jcc);
16371 %}
16372
16373 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
16374 match(If cop cmp);
16375 effect(USE labl);
16376
16377 ins_cost(200);
16378 format %{ "j$cop,u $labl" %}
16379 size(6);
16380 ins_encode %{
16381 Label* L = $labl$$label;
16382 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16383 %}
16384 ins_pipe(pipe_jcc);
16385 %}
16386
16387 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
16388 match(If cop cmp);
16389 effect(USE labl);
16390
16391 ins_cost(200);
16392 format %{ $$template
16393 if ($cop$$cmpcode == Assembler::notEqual) {
16394 $$emit$$"jp,u $labl\n\t"
16395 $$emit$$"j$cop,u $labl"
16396 } else {
16397 $$emit$$"jp,u done\n\t"
16398 $$emit$$"j$cop,u $labl\n\t"
16399 $$emit$$"done:"
16400 }
16401 %}
16402 ins_encode %{
16403 Label* l = $labl$$label;
16404 if ($cop$$cmpcode == Assembler::notEqual) {
16405 __ jcc(Assembler::parity, *l, false);
16406 __ jcc(Assembler::notEqual, *l, false);
16407 } else if ($cop$$cmpcode == Assembler::equal) {
16408 Label done;
16409 __ jccb(Assembler::parity, done);
16410 __ jcc(Assembler::equal, *l, false);
16411 __ bind(done);
16412 } else {
16413 ShouldNotReachHere();
16414 }
16415 %}
16416 ins_pipe(pipe_jcc);
16417 %}
16418
16419 // Jump Direct Conditional - using signed and unsigned comparison
16420 instruct jmpConUCFE(cmpOpUCFE cop, rFlagsRegUCFE cmp, label labl) %{
16421 match(If cop cmp);
16422 effect(USE labl);
16423
16424 ins_cost(200);
16425 format %{ "j$cop,su $labl" %}
16426 size(6);
16427 ins_encode %{
16428 Label* L = $labl$$label;
16429 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16430 %}
16431 ins_pipe(pipe_jcc);
16432 %}
16433
16434 // ============================================================================
16435 // The 2nd slow-half of a subtype check. Scan the subklass's 2ndary
16436 // superklass array for an instance of the superklass. Set a hidden
16437 // internal cache on a hit (cache is checked with exposed code in
16438 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16439 // encoding ALSO sets flags.
16440
16441 instruct partialSubtypeCheck(rdi_RegP result,
16442 rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
16443 rFlagsReg cr)
16444 %{
16445 match(Set result (PartialSubtypeCheck sub super));
16446 predicate(!UseSecondarySupersTable);
16447 effect(KILL rcx, KILL cr);
16448
16449 ins_cost(1100); // slightly larger than the next version
16450 format %{ "movq rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
16451 "movl rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
16452 "addq rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
16453 "repne scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
16454 "jne,s miss\t\t# Missed: rdi not-zero\n\t"
16455 "movq [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
16456 "xorq $result, $result\t\t Hit: rdi zero\n\t"
16457 "miss:\t" %}
16458
16459 ins_encode %{
16460 Label miss;
16461 // NB: Callers may assume that, when $result is a valid register,
16462 // check_klass_subtype_slow_path_linear sets it to a nonzero
16463 // value.
16464 __ check_klass_subtype_slow_path_linear($sub$$Register, $super$$Register,
16465 $rcx$$Register, $result$$Register,
16466 nullptr, &miss,
16467 /*set_cond_codes:*/ true);
16468 __ xorptr($result$$Register, $result$$Register);
16469 __ bind(miss);
16470 %}
16471
16472 ins_pipe(pipe_slow);
16473 %}
16474
16475 // ============================================================================
16476 // Two versions of hashtable-based partialSubtypeCheck, both used when
16477 // we need to search for a super class in the secondary supers array.
16478 // The first is used when we don't know _a priori_ the class being
16479 // searched for. The second, far more common, is used when we do know:
16480 // this is used for instanceof, checkcast, and any case where C2 can
16481 // determine it by constant propagation.
16482
16483 instruct partialSubtypeCheckVarSuper(rsi_RegP sub, rax_RegP super, rdi_RegP result,
16484 rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
16485 rFlagsReg cr)
16486 %{
16487 match(Set result (PartialSubtypeCheck sub super));
16488 predicate(UseSecondarySupersTable);
16489 effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
16490
16491 ins_cost(1000);
16492 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16493
16494 ins_encode %{
16495 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register, $temp1$$Register, $temp2$$Register,
16496 $temp3$$Register, $temp4$$Register, $result$$Register);
16497 %}
16498
16499 ins_pipe(pipe_slow);
16500 %}
16501
16502 instruct partialSubtypeCheckConstSuper(rsi_RegP sub, rax_RegP super_reg, immP super_con, rdi_RegP result,
16503 rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
16504 rFlagsReg cr)
16505 %{
16506 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16507 predicate(UseSecondarySupersTable);
16508 effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
16509
16510 ins_cost(700); // smaller than the next version
16511 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16512
16513 ins_encode %{
16514 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16515 if (InlineSecondarySupersTest) {
16516 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register, $temp1$$Register, $temp2$$Register,
16517 $temp3$$Register, $temp4$$Register, $result$$Register,
16518 super_klass_slot);
16519 } else {
16520 __ call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16521 }
16522 %}
16523
16524 ins_pipe(pipe_slow);
16525 %}
16526
16527 // ============================================================================
16528 // Branch Instructions -- short offset versions
16529 //
16530 // These instructions are used to replace jumps of a long offset (the default
16531 // match) with jumps of a shorter offset. These instructions are all tagged
16532 // with the ins_short_branch attribute, which causes the ADLC to suppress the
16533 // match rules in general matching. Instead, the ADLC generates a conversion
16534 // method in the MachNode which can be used to do in-place replacement of the
16535 // long variant with the shorter variant. The compiler will determine if a
16536 // branch can be taken by the is_short_branch_offset() predicate in the machine
16537 // specific code section of the file.
16538
16539 // Jump Direct - Label defines a relative address from JMP+1
16540 instruct jmpDir_short(label labl) %{
16541 match(Goto);
16542 effect(USE labl);
16543
16544 ins_cost(300);
16545 format %{ "jmp,s $labl" %}
16546 size(2);
16547 ins_encode %{
16548 Label* L = $labl$$label;
16549 __ jmpb(*L);
16550 %}
16551 ins_pipe(pipe_jmp);
16552 ins_short_branch(1);
16553 %}
16554
16555 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16556 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
16557 match(If cop cr);
16558 effect(USE labl);
16559
16560 ins_cost(300);
16561 format %{ "j$cop,s $labl" %}
16562 size(2);
16563 ins_encode %{
16564 Label* L = $labl$$label;
16565 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16566 %}
16567 ins_pipe(pipe_jcc);
16568 ins_short_branch(1);
16569 %}
16570
16571 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16572 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
16573 match(CountedLoopEnd cop cr);
16574 effect(USE labl);
16575
16576 ins_cost(300);
16577 format %{ "j$cop,s $labl\t# loop end" %}
16578 size(2);
16579 ins_encode %{
16580 Label* L = $labl$$label;
16581 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16582 %}
16583 ins_pipe(pipe_jcc);
16584 ins_short_branch(1);
16585 %}
16586
16587 // Jump Direct Conditional - using unsigned comparison
16588 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
16589 match(If cop cmp);
16590 effect(USE labl);
16591
16592 ins_cost(300);
16593 format %{ "j$cop,us $labl" %}
16594 size(2);
16595 ins_encode %{
16596 Label* L = $labl$$label;
16597 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16598 %}
16599 ins_pipe(pipe_jcc);
16600 ins_short_branch(1);
16601 %}
16602
16603 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
16604 match(If cop cmp);
16605 effect(USE labl);
16606
16607 ins_cost(300);
16608 format %{ "j$cop,us $labl" %}
16609 size(2);
16610 ins_encode %{
16611 Label* L = $labl$$label;
16612 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16613 %}
16614 ins_pipe(pipe_jcc);
16615 ins_short_branch(1);
16616 %}
16617
16618 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
16619 match(If cop cmp);
16620 effect(USE labl);
16621
16622 ins_cost(300);
16623 format %{ $$template
16624 if ($cop$$cmpcode == Assembler::notEqual) {
16625 $$emit$$"jp,u,s $labl\n\t"
16626 $$emit$$"j$cop,u,s $labl"
16627 } else {
16628 $$emit$$"jp,u,s done\n\t"
16629 $$emit$$"j$cop,u,s $labl\n\t"
16630 $$emit$$"done:"
16631 }
16632 %}
16633 size(4);
16634 ins_encode %{
16635 Label* l = $labl$$label;
16636 if ($cop$$cmpcode == Assembler::notEqual) {
16637 __ jccb(Assembler::parity, *l);
16638 __ jccb(Assembler::notEqual, *l);
16639 } else if ($cop$$cmpcode == Assembler::equal) {
16640 Label done;
16641 __ jccb(Assembler::parity, done);
16642 __ jccb(Assembler::equal, *l);
16643 __ bind(done);
16644 } else {
16645 ShouldNotReachHere();
16646 }
16647 %}
16648 ins_pipe(pipe_jcc);
16649 ins_short_branch(1);
16650 %}
16651
16652 // Jump Direct Conditional - using signed and unsigned comparison
16653 instruct jmpConUCFE_short(cmpOpUCFE cop, rFlagsRegUCFE cmp, label labl) %{
16654 match(If cop cmp);
16655 effect(USE labl);
16656
16657 ins_cost(300);
16658 format %{ "j$cop,sus $labl" %}
16659 size(2);
16660 ins_encode %{
16661 Label* L = $labl$$label;
16662 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16663 %}
16664 ins_pipe(pipe_jcc);
16665 ins_short_branch(1);
16666 %}
16667
16668 // ============================================================================
16669 // inlined locking and unlocking
16670
16671 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
16672 match(Set cr (FastLock object box));
16673 effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
16674 ins_cost(300);
16675 format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
16676 ins_encode %{
16677 __ fast_lock($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
16678 %}
16679 ins_pipe(pipe_slow);
16680 %}
16681
16682 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
16683 match(Set cr (FastUnlock object rax_reg));
16684 effect(TEMP tmp, USE_KILL rax_reg);
16685 ins_cost(300);
16686 format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
16687 ins_encode %{
16688 __ fast_unlock($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
16689 %}
16690 ins_pipe(pipe_slow);
16691 %}
16692
16693
16694 // ============================================================================
16695 // Safepoint Instructions
16696 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
16697 %{
16698 match(SafePoint poll);
16699 effect(KILL cr, USE poll);
16700
16701 format %{ "testl rax, [$poll]\t"
16702 "# Safepoint: poll for GC" %}
16703 ins_cost(125);
16704 ins_encode %{
16705 __ relocate(relocInfo::poll_type);
16706 address pre_pc = __ pc();
16707 __ testl(rax, Address($poll$$Register, 0));
16708 assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
16709 %}
16710 ins_pipe(ialu_reg_mem);
16711 %}
16712
16713 instruct mask_all_evexL(kReg dst, rRegL src) %{
16714 match(Set dst (MaskAll src));
16715 format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
16716 ins_encode %{
16717 int mask_len = Matcher::vector_length(this);
16718 __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
16719 %}
16720 ins_pipe( pipe_slow );
16721 %}
16722
16723 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
16724 predicate(Matcher::vector_length(n) > 32);
16725 match(Set dst (MaskAll src));
16726 effect(TEMP tmp);
16727 format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
16728 ins_encode %{
16729 int mask_len = Matcher::vector_length(this);
16730 __ movslq($tmp$$Register, $src$$Register);
16731 __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
16732 %}
16733 ins_pipe( pipe_slow );
16734 %}
16735
16736 // ============================================================================
16737 // Procedure Call/Return Instructions
16738 // Call Java Static Instruction
16739 // Note: If this code changes, the corresponding ret_addr_offset() and
16740 // compute_padding() functions will have to be adjusted.
16741 instruct CallStaticJavaDirect(method meth) %{
16742 match(CallStaticJava);
16743 effect(USE meth);
16744
16745 ins_cost(300);
16746 format %{ "call,static " %}
16747 opcode(0xE8); /* E8 cd */
16748 ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
16749 ins_pipe(pipe_slow);
16750 ins_alignment(4);
16751 %}
16752
16753 // Call Java Dynamic Instruction
16754 // Note: If this code changes, the corresponding ret_addr_offset() and
16755 // compute_padding() functions will have to be adjusted.
16756 instruct CallDynamicJavaDirect(method meth)
16757 %{
16758 match(CallDynamicJava);
16759 effect(USE meth);
16760
16761 ins_cost(300);
16762 format %{ "movq rax, #Universe::non_oop_word()\n\t"
16763 "call,dynamic " %}
16764 ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
16765 ins_pipe(pipe_slow);
16766 ins_alignment(4);
16767 %}
16768
16769 // Call Runtime Instruction
16770 instruct CallRuntimeDirect(method meth)
16771 %{
16772 match(CallRuntime);
16773 effect(USE meth);
16774
16775 ins_cost(300);
16776 format %{ "call,runtime " %}
16777 ins_encode(clear_avx, Java_To_Runtime(meth));
16778 ins_pipe(pipe_slow);
16779 %}
16780
16781 // Call runtime without safepoint
16782 instruct CallLeafDirect(method meth)
16783 %{
16784 match(CallLeaf);
16785 effect(USE meth);
16786
16787 ins_cost(300);
16788 format %{ "call_leaf,runtime " %}
16789 ins_encode(clear_avx, Java_To_Runtime(meth));
16790 ins_pipe(pipe_slow);
16791 %}
16792
16793 // Call runtime without safepoint and with vector arguments
16794 instruct CallLeafDirectVector(method meth)
16795 %{
16796 match(CallLeafVector);
16797 effect(USE meth);
16798
16799 ins_cost(300);
16800 format %{ "call_leaf,vector " %}
16801 ins_encode(Java_To_Runtime(meth));
16802 ins_pipe(pipe_slow);
16803 %}
16804
16805 // Call runtime without safepoint
16806 instruct CallLeafNoFPDirect(method meth)
16807 %{
16808 match(CallLeafNoFP);
16809 effect(USE meth);
16810
16811 ins_cost(300);
16812 format %{ "call_leaf_nofp,runtime " %}
16813 ins_encode(clear_avx, Java_To_Runtime(meth));
16814 ins_pipe(pipe_slow);
16815 %}
16816
16817 // Return Instruction
16818 // Remove the return address & jump to it.
16819 // Notice: We always emit a nop after a ret to make sure there is room
16820 // for safepoint patching
16821 instruct Ret()
16822 %{
16823 match(Return);
16824
16825 format %{ "ret" %}
16826 ins_encode %{
16827 __ ret(0);
16828 %}
16829 ins_pipe(pipe_jmp);
16830 %}
16831
16832 // Tail Call; Jump from runtime stub to Java code.
16833 // Also known as an 'interprocedural jump'.
16834 // Target of jump will eventually return to caller.
16835 // TailJump below removes the return address.
16836 // Don't use rbp for 'jump_target' because a MachEpilogNode has already been
16837 // emitted just above the TailCall which has reset rbp to the caller state.
16838 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
16839 %{
16840 match(TailCall jump_target method_ptr);
16841
16842 ins_cost(300);
16843 format %{ "jmp $jump_target\t# rbx holds method" %}
16844 ins_encode %{
16845 __ jmp($jump_target$$Register);
16846 %}
16847 ins_pipe(pipe_jmp);
16848 %}
16849
16850 // Tail Jump; remove the return address; jump to target.
16851 // TailCall above leaves the return address around.
16852 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
16853 %{
16854 match(TailJump jump_target ex_oop);
16855
16856 ins_cost(300);
16857 format %{ "popq rdx\t# pop return address\n\t"
16858 "jmp $jump_target" %}
16859 ins_encode %{
16860 __ popq(as_Register(RDX_enc));
16861 __ jmp($jump_target$$Register);
16862 %}
16863 ins_pipe(pipe_jmp);
16864 %}
16865
16866 // Forward exception.
16867 instruct ForwardExceptionjmp()
16868 %{
16869 match(ForwardException);
16870
16871 format %{ "jmp forward_exception_stub" %}
16872 ins_encode %{
16873 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()), noreg);
16874 %}
16875 ins_pipe(pipe_jmp);
16876 %}
16877
16878 // Create exception oop: created by stack-crawling runtime code.
16879 // Created exception is now available to this handler, and is setup
16880 // just prior to jumping to this handler. No code emitted.
16881 instruct CreateException(rax_RegP ex_oop)
16882 %{
16883 match(Set ex_oop (CreateEx));
16884
16885 size(0);
16886 // use the following format syntax
16887 format %{ "# exception oop is in rax; no code emitted" %}
16888 ins_encode();
16889 ins_pipe(empty);
16890 %}
16891
16892 // Rethrow exception:
16893 // The exception oop will come in the first argument position.
16894 // Then JUMP (not call) to the rethrow stub code.
16895 instruct RethrowException()
16896 %{
16897 match(Rethrow);
16898
16899 // use the following format syntax
16900 format %{ "jmp rethrow_stub" %}
16901 ins_encode %{
16902 __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
16903 %}
16904 ins_pipe(pipe_jmp);
16905 %}
16906
16907 // ============================================================================
16908 // This name is KNOWN by the ADLC and cannot be changed.
16909 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16910 // for this guy.
16911 instruct tlsLoadP(r15_RegP dst) %{
16912 match(Set dst (ThreadLocal));
16913 effect(DEF dst);
16914
16915 size(0);
16916 format %{ "# TLS is in R15" %}
16917 ins_encode( /*empty encoding*/ );
16918 ins_pipe(ialu_reg_reg);
16919 %}
16920
16921 instruct addF_reg(regF dst, regF src) %{
16922 predicate(UseAVX == 0);
16923 match(Set dst (AddF dst src));
16924
16925 format %{ "addss $dst, $src" %}
16926 ins_cost(150);
16927 ins_encode %{
16928 __ addss($dst$$XMMRegister, $src$$XMMRegister);
16929 %}
16930 ins_pipe(pipe_slow);
16931 %}
16932
16933 instruct addF_mem(regF dst, memory src) %{
16934 predicate(UseAVX == 0);
16935 match(Set dst (AddF dst (LoadF src)));
16936
16937 format %{ "addss $dst, $src" %}
16938 ins_cost(150);
16939 ins_encode %{
16940 __ addss($dst$$XMMRegister, $src$$Address);
16941 %}
16942 ins_pipe(pipe_slow);
16943 %}
16944
16945 instruct addF_imm(regF dst, immF con) %{
16946 predicate(UseAVX == 0);
16947 match(Set dst (AddF dst con));
16948 format %{ "addss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
16949 ins_cost(150);
16950 ins_encode %{
16951 __ addss($dst$$XMMRegister, $constantaddress($con));
16952 %}
16953 ins_pipe(pipe_slow);
16954 %}
16955
16956 instruct addF_reg_reg(regF dst, regF src1, regF src2) %{
16957 predicate(UseAVX > 0);
16958 match(Set dst (AddF src1 src2));
16959
16960 format %{ "vaddss $dst, $src1, $src2" %}
16961 ins_cost(150);
16962 ins_encode %{
16963 __ vaddss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
16964 %}
16965 ins_pipe(pipe_slow);
16966 %}
16967
16968 instruct addF_reg_mem(regF dst, regF src1, memory src2) %{
16969 predicate(UseAVX > 0);
16970 match(Set dst (AddF src1 (LoadF src2)));
16971
16972 format %{ "vaddss $dst, $src1, $src2" %}
16973 ins_cost(150);
16974 ins_encode %{
16975 __ vaddss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
16976 %}
16977 ins_pipe(pipe_slow);
16978 %}
16979
16980 instruct addF_reg_imm(regF dst, regF src, immF con) %{
16981 predicate(UseAVX > 0);
16982 match(Set dst (AddF src con));
16983
16984 format %{ "vaddss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
16985 ins_cost(150);
16986 ins_encode %{
16987 __ vaddss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
16988 %}
16989 ins_pipe(pipe_slow);
16990 %}
16991
16992 instruct addD_reg(regD dst, regD src) %{
16993 predicate(UseAVX == 0);
16994 match(Set dst (AddD dst src));
16995
16996 format %{ "addsd $dst, $src" %}
16997 ins_cost(150);
16998 ins_encode %{
16999 __ addsd($dst$$XMMRegister, $src$$XMMRegister);
17000 %}
17001 ins_pipe(pipe_slow);
17002 %}
17003
17004 instruct addD_mem(regD dst, memory src) %{
17005 predicate(UseAVX == 0);
17006 match(Set dst (AddD dst (LoadD src)));
17007
17008 format %{ "addsd $dst, $src" %}
17009 ins_cost(150);
17010 ins_encode %{
17011 __ addsd($dst$$XMMRegister, $src$$Address);
17012 %}
17013 ins_pipe(pipe_slow);
17014 %}
17015
17016 instruct addD_imm(regD dst, immD con) %{
17017 predicate(UseAVX == 0);
17018 match(Set dst (AddD dst con));
17019 format %{ "addsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17020 ins_cost(150);
17021 ins_encode %{
17022 __ addsd($dst$$XMMRegister, $constantaddress($con));
17023 %}
17024 ins_pipe(pipe_slow);
17025 %}
17026
17027 instruct addD_reg_reg(regD dst, regD src1, regD src2) %{
17028 predicate(UseAVX > 0);
17029 match(Set dst (AddD src1 src2));
17030
17031 format %{ "vaddsd $dst, $src1, $src2" %}
17032 ins_cost(150);
17033 ins_encode %{
17034 __ vaddsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17035 %}
17036 ins_pipe(pipe_slow);
17037 %}
17038
17039 instruct addD_reg_mem(regD dst, regD src1, memory src2) %{
17040 predicate(UseAVX > 0);
17041 match(Set dst (AddD src1 (LoadD src2)));
17042
17043 format %{ "vaddsd $dst, $src1, $src2" %}
17044 ins_cost(150);
17045 ins_encode %{
17046 __ vaddsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17047 %}
17048 ins_pipe(pipe_slow);
17049 %}
17050
17051 instruct addD_reg_imm(regD dst, regD src, immD con) %{
17052 predicate(UseAVX > 0);
17053 match(Set dst (AddD src con));
17054
17055 format %{ "vaddsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17056 ins_cost(150);
17057 ins_encode %{
17058 __ vaddsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17059 %}
17060 ins_pipe(pipe_slow);
17061 %}
17062
17063 instruct subF_reg(regF dst, regF src) %{
17064 predicate(UseAVX == 0);
17065 match(Set dst (SubF dst src));
17066
17067 format %{ "subss $dst, $src" %}
17068 ins_cost(150);
17069 ins_encode %{
17070 __ subss($dst$$XMMRegister, $src$$XMMRegister);
17071 %}
17072 ins_pipe(pipe_slow);
17073 %}
17074
17075 instruct subF_mem(regF dst, memory src) %{
17076 predicate(UseAVX == 0);
17077 match(Set dst (SubF dst (LoadF src)));
17078
17079 format %{ "subss $dst, $src" %}
17080 ins_cost(150);
17081 ins_encode %{
17082 __ subss($dst$$XMMRegister, $src$$Address);
17083 %}
17084 ins_pipe(pipe_slow);
17085 %}
17086
17087 instruct subF_imm(regF dst, immF con) %{
17088 predicate(UseAVX == 0);
17089 match(Set dst (SubF dst con));
17090 format %{ "subss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
17091 ins_cost(150);
17092 ins_encode %{
17093 __ subss($dst$$XMMRegister, $constantaddress($con));
17094 %}
17095 ins_pipe(pipe_slow);
17096 %}
17097
17098 instruct subF_reg_reg(regF dst, regF src1, regF src2) %{
17099 predicate(UseAVX > 0);
17100 match(Set dst (SubF src1 src2));
17101
17102 format %{ "vsubss $dst, $src1, $src2" %}
17103 ins_cost(150);
17104 ins_encode %{
17105 __ vsubss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17106 %}
17107 ins_pipe(pipe_slow);
17108 %}
17109
17110 instruct subF_reg_mem(regF dst, regF src1, memory src2) %{
17111 predicate(UseAVX > 0);
17112 match(Set dst (SubF src1 (LoadF src2)));
17113
17114 format %{ "vsubss $dst, $src1, $src2" %}
17115 ins_cost(150);
17116 ins_encode %{
17117 __ vsubss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17118 %}
17119 ins_pipe(pipe_slow);
17120 %}
17121
17122 instruct subF_reg_imm(regF dst, regF src, immF con) %{
17123 predicate(UseAVX > 0);
17124 match(Set dst (SubF src con));
17125
17126 format %{ "vsubss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
17127 ins_cost(150);
17128 ins_encode %{
17129 __ vsubss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17130 %}
17131 ins_pipe(pipe_slow);
17132 %}
17133
17134 instruct subD_reg(regD dst, regD src) %{
17135 predicate(UseAVX == 0);
17136 match(Set dst (SubD dst src));
17137
17138 format %{ "subsd $dst, $src" %}
17139 ins_cost(150);
17140 ins_encode %{
17141 __ subsd($dst$$XMMRegister, $src$$XMMRegister);
17142 %}
17143 ins_pipe(pipe_slow);
17144 %}
17145
17146 instruct subD_mem(regD dst, memory src) %{
17147 predicate(UseAVX == 0);
17148 match(Set dst (SubD dst (LoadD src)));
17149
17150 format %{ "subsd $dst, $src" %}
17151 ins_cost(150);
17152 ins_encode %{
17153 __ subsd($dst$$XMMRegister, $src$$Address);
17154 %}
17155 ins_pipe(pipe_slow);
17156 %}
17157
17158 instruct subD_imm(regD dst, immD con) %{
17159 predicate(UseAVX == 0);
17160 match(Set dst (SubD dst con));
17161 format %{ "subsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17162 ins_cost(150);
17163 ins_encode %{
17164 __ subsd($dst$$XMMRegister, $constantaddress($con));
17165 %}
17166 ins_pipe(pipe_slow);
17167 %}
17168
17169 instruct subD_reg_reg(regD dst, regD src1, regD src2) %{
17170 predicate(UseAVX > 0);
17171 match(Set dst (SubD src1 src2));
17172
17173 format %{ "vsubsd $dst, $src1, $src2" %}
17174 ins_cost(150);
17175 ins_encode %{
17176 __ vsubsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17177 %}
17178 ins_pipe(pipe_slow);
17179 %}
17180
17181 instruct subD_reg_mem(regD dst, regD src1, memory src2) %{
17182 predicate(UseAVX > 0);
17183 match(Set dst (SubD src1 (LoadD src2)));
17184
17185 format %{ "vsubsd $dst, $src1, $src2" %}
17186 ins_cost(150);
17187 ins_encode %{
17188 __ vsubsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17189 %}
17190 ins_pipe(pipe_slow);
17191 %}
17192
17193 instruct subD_reg_imm(regD dst, regD src, immD con) %{
17194 predicate(UseAVX > 0);
17195 match(Set dst (SubD src con));
17196
17197 format %{ "vsubsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17198 ins_cost(150);
17199 ins_encode %{
17200 __ vsubsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17201 %}
17202 ins_pipe(pipe_slow);
17203 %}
17204
17205 instruct mulF_reg(regF dst, regF src) %{
17206 predicate(UseAVX == 0);
17207 match(Set dst (MulF dst src));
17208
17209 format %{ "mulss $dst, $src" %}
17210 ins_cost(150);
17211 ins_encode %{
17212 __ mulss($dst$$XMMRegister, $src$$XMMRegister);
17213 %}
17214 ins_pipe(pipe_slow);
17215 %}
17216
17217 instruct mulF_mem(regF dst, memory src) %{
17218 predicate(UseAVX == 0);
17219 match(Set dst (MulF dst (LoadF src)));
17220
17221 format %{ "mulss $dst, $src" %}
17222 ins_cost(150);
17223 ins_encode %{
17224 __ mulss($dst$$XMMRegister, $src$$Address);
17225 %}
17226 ins_pipe(pipe_slow);
17227 %}
17228
17229 instruct mulF_imm(regF dst, immF con) %{
17230 predicate(UseAVX == 0);
17231 match(Set dst (MulF dst con));
17232 format %{ "mulss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
17233 ins_cost(150);
17234 ins_encode %{
17235 __ mulss($dst$$XMMRegister, $constantaddress($con));
17236 %}
17237 ins_pipe(pipe_slow);
17238 %}
17239
17240 instruct mulF_reg_reg(regF dst, regF src1, regF src2) %{
17241 predicate(UseAVX > 0);
17242 match(Set dst (MulF src1 src2));
17243
17244 format %{ "vmulss $dst, $src1, $src2" %}
17245 ins_cost(150);
17246 ins_encode %{
17247 __ vmulss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17248 %}
17249 ins_pipe(pipe_slow);
17250 %}
17251
17252 instruct mulF_reg_mem(regF dst, regF src1, memory src2) %{
17253 predicate(UseAVX > 0);
17254 match(Set dst (MulF src1 (LoadF src2)));
17255
17256 format %{ "vmulss $dst, $src1, $src2" %}
17257 ins_cost(150);
17258 ins_encode %{
17259 __ vmulss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17260 %}
17261 ins_pipe(pipe_slow);
17262 %}
17263
17264 instruct mulF_reg_imm(regF dst, regF src, immF con) %{
17265 predicate(UseAVX > 0);
17266 match(Set dst (MulF src con));
17267
17268 format %{ "vmulss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
17269 ins_cost(150);
17270 ins_encode %{
17271 __ vmulss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17272 %}
17273 ins_pipe(pipe_slow);
17274 %}
17275
17276 instruct mulD_reg(regD dst, regD src) %{
17277 predicate(UseAVX == 0);
17278 match(Set dst (MulD dst src));
17279
17280 format %{ "mulsd $dst, $src" %}
17281 ins_cost(150);
17282 ins_encode %{
17283 __ mulsd($dst$$XMMRegister, $src$$XMMRegister);
17284 %}
17285 ins_pipe(pipe_slow);
17286 %}
17287
17288 instruct mulD_mem(regD dst, memory src) %{
17289 predicate(UseAVX == 0);
17290 match(Set dst (MulD dst (LoadD src)));
17291
17292 format %{ "mulsd $dst, $src" %}
17293 ins_cost(150);
17294 ins_encode %{
17295 __ mulsd($dst$$XMMRegister, $src$$Address);
17296 %}
17297 ins_pipe(pipe_slow);
17298 %}
17299
17300 instruct mulD_imm(regD dst, immD con) %{
17301 predicate(UseAVX == 0);
17302 match(Set dst (MulD dst con));
17303 format %{ "mulsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17304 ins_cost(150);
17305 ins_encode %{
17306 __ mulsd($dst$$XMMRegister, $constantaddress($con));
17307 %}
17308 ins_pipe(pipe_slow);
17309 %}
17310
17311 instruct mulD_reg_reg(regD dst, regD src1, regD src2) %{
17312 predicate(UseAVX > 0);
17313 match(Set dst (MulD src1 src2));
17314
17315 format %{ "vmulsd $dst, $src1, $src2" %}
17316 ins_cost(150);
17317 ins_encode %{
17318 __ vmulsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17319 %}
17320 ins_pipe(pipe_slow);
17321 %}
17322
17323 instruct mulD_reg_mem(regD dst, regD src1, memory src2) %{
17324 predicate(UseAVX > 0);
17325 match(Set dst (MulD src1 (LoadD src2)));
17326
17327 format %{ "vmulsd $dst, $src1, $src2" %}
17328 ins_cost(150);
17329 ins_encode %{
17330 __ vmulsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17331 %}
17332 ins_pipe(pipe_slow);
17333 %}
17334
17335 instruct mulD_reg_imm(regD dst, regD src, immD con) %{
17336 predicate(UseAVX > 0);
17337 match(Set dst (MulD src con));
17338
17339 format %{ "vmulsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17340 ins_cost(150);
17341 ins_encode %{
17342 __ vmulsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17343 %}
17344 ins_pipe(pipe_slow);
17345 %}
17346
17347 instruct divF_reg(regF dst, regF src) %{
17348 predicate(UseAVX == 0);
17349 match(Set dst (DivF dst src));
17350
17351 format %{ "divss $dst, $src" %}
17352 ins_cost(150);
17353 ins_encode %{
17354 __ divss($dst$$XMMRegister, $src$$XMMRegister);
17355 %}
17356 ins_pipe(pipe_slow);
17357 %}
17358
17359 instruct divF_mem(regF dst, memory src) %{
17360 predicate(UseAVX == 0);
17361 match(Set dst (DivF dst (LoadF src)));
17362
17363 format %{ "divss $dst, $src" %}
17364 ins_cost(150);
17365 ins_encode %{
17366 __ divss($dst$$XMMRegister, $src$$Address);
17367 %}
17368 ins_pipe(pipe_slow);
17369 %}
17370
17371 instruct divF_imm(regF dst, immF con) %{
17372 predicate(UseAVX == 0);
17373 match(Set dst (DivF dst con));
17374 format %{ "divss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
17375 ins_cost(150);
17376 ins_encode %{
17377 __ divss($dst$$XMMRegister, $constantaddress($con));
17378 %}
17379 ins_pipe(pipe_slow);
17380 %}
17381
17382 instruct divF_reg_reg(regF dst, regF src1, regF src2) %{
17383 predicate(UseAVX > 0);
17384 match(Set dst (DivF src1 src2));
17385
17386 format %{ "vdivss $dst, $src1, $src2" %}
17387 ins_cost(150);
17388 ins_encode %{
17389 __ vdivss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17390 %}
17391 ins_pipe(pipe_slow);
17392 %}
17393
17394 instruct divF_reg_mem(regF dst, regF src1, memory src2) %{
17395 predicate(UseAVX > 0);
17396 match(Set dst (DivF src1 (LoadF src2)));
17397
17398 format %{ "vdivss $dst, $src1, $src2" %}
17399 ins_cost(150);
17400 ins_encode %{
17401 __ vdivss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17402 %}
17403 ins_pipe(pipe_slow);
17404 %}
17405
17406 instruct divF_reg_imm(regF dst, regF src, immF con) %{
17407 predicate(UseAVX > 0);
17408 match(Set dst (DivF src con));
17409
17410 format %{ "vdivss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
17411 ins_cost(150);
17412 ins_encode %{
17413 __ vdivss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17414 %}
17415 ins_pipe(pipe_slow);
17416 %}
17417
17418 instruct divD_reg(regD dst, regD src) %{
17419 predicate(UseAVX == 0);
17420 match(Set dst (DivD dst src));
17421
17422 format %{ "divsd $dst, $src" %}
17423 ins_cost(150);
17424 ins_encode %{
17425 __ divsd($dst$$XMMRegister, $src$$XMMRegister);
17426 %}
17427 ins_pipe(pipe_slow);
17428 %}
17429
17430 instruct divD_mem(regD dst, memory src) %{
17431 predicate(UseAVX == 0);
17432 match(Set dst (DivD dst (LoadD src)));
17433
17434 format %{ "divsd $dst, $src" %}
17435 ins_cost(150);
17436 ins_encode %{
17437 __ divsd($dst$$XMMRegister, $src$$Address);
17438 %}
17439 ins_pipe(pipe_slow);
17440 %}
17441
17442 instruct divD_imm(regD dst, immD con) %{
17443 predicate(UseAVX == 0);
17444 match(Set dst (DivD dst con));
17445 format %{ "divsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17446 ins_cost(150);
17447 ins_encode %{
17448 __ divsd($dst$$XMMRegister, $constantaddress($con));
17449 %}
17450 ins_pipe(pipe_slow);
17451 %}
17452
17453 instruct divD_reg_reg(regD dst, regD src1, regD src2) %{
17454 predicate(UseAVX > 0);
17455 match(Set dst (DivD src1 src2));
17456
17457 format %{ "vdivsd $dst, $src1, $src2" %}
17458 ins_cost(150);
17459 ins_encode %{
17460 __ vdivsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17461 %}
17462 ins_pipe(pipe_slow);
17463 %}
17464
17465 instruct divD_reg_mem(regD dst, regD src1, memory src2) %{
17466 predicate(UseAVX > 0);
17467 match(Set dst (DivD src1 (LoadD src2)));
17468
17469 format %{ "vdivsd $dst, $src1, $src2" %}
17470 ins_cost(150);
17471 ins_encode %{
17472 __ vdivsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17473 %}
17474 ins_pipe(pipe_slow);
17475 %}
17476
17477 instruct divD_reg_imm(regD dst, regD src, immD con) %{
17478 predicate(UseAVX > 0);
17479 match(Set dst (DivD src con));
17480
17481 format %{ "vdivsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17482 ins_cost(150);
17483 ins_encode %{
17484 __ vdivsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17485 %}
17486 ins_pipe(pipe_slow);
17487 %}
17488
17489 instruct absF_reg(regF dst) %{
17490 predicate(UseAVX == 0);
17491 match(Set dst (AbsF dst));
17492 ins_cost(150);
17493 format %{ "andps $dst, [0x7fffffff]\t# abs float by sign masking" %}
17494 ins_encode %{
17495 __ andps($dst$$XMMRegister, ExternalAddress(float_signmask()));
17496 %}
17497 ins_pipe(pipe_slow);
17498 %}
17499
17500 instruct absF_reg_reg(vlRegF dst, vlRegF src) %{
17501 predicate(UseAVX > 0);
17502 match(Set dst (AbsF src));
17503 ins_cost(150);
17504 format %{ "vandps $dst, $src, [0x7fffffff]\t# abs float by sign masking" %}
17505 ins_encode %{
17506 int vlen_enc = Assembler::AVX_128bit;
17507 __ vandps($dst$$XMMRegister, $src$$XMMRegister,
17508 ExternalAddress(float_signmask()), vlen_enc);
17509 %}
17510 ins_pipe(pipe_slow);
17511 %}
17512
17513 instruct absD_reg(regD dst) %{
17514 predicate(UseAVX == 0);
17515 match(Set dst (AbsD dst));
17516 ins_cost(150);
17517 format %{ "andpd $dst, [0x7fffffffffffffff]\t"
17518 "# abs double by sign masking" %}
17519 ins_encode %{
17520 __ andpd($dst$$XMMRegister, ExternalAddress(double_signmask()));
17521 %}
17522 ins_pipe(pipe_slow);
17523 %}
17524
17525 instruct absD_reg_reg(vlRegD dst, vlRegD src) %{
17526 predicate(UseAVX > 0);
17527 match(Set dst (AbsD src));
17528 ins_cost(150);
17529 format %{ "vandpd $dst, $src, [0x7fffffffffffffff]\t"
17530 "# abs double by sign masking" %}
17531 ins_encode %{
17532 int vlen_enc = Assembler::AVX_128bit;
17533 __ vandpd($dst$$XMMRegister, $src$$XMMRegister,
17534 ExternalAddress(double_signmask()), vlen_enc);
17535 %}
17536 ins_pipe(pipe_slow);
17537 %}
17538
17539 instruct negF_reg(regF dst) %{
17540 predicate(UseAVX == 0);
17541 match(Set dst (NegF dst));
17542 ins_cost(150);
17543 format %{ "xorps $dst, [0x80000000]\t# neg float by sign flipping" %}
17544 ins_encode %{
17545 __ xorps($dst$$XMMRegister, ExternalAddress(float_signflip()));
17546 %}
17547 ins_pipe(pipe_slow);
17548 %}
17549
17550 instruct negF_reg_reg(vlRegF dst, vlRegF src) %{
17551 predicate(UseAVX > 0);
17552 match(Set dst (NegF src));
17553 ins_cost(150);
17554 format %{ "vnegatess $dst, $src, [0x80000000]\t# neg float by sign flipping" %}
17555 ins_encode %{
17556 __ vnegatess($dst$$XMMRegister, $src$$XMMRegister,
17557 ExternalAddress(float_signflip()));
17558 %}
17559 ins_pipe(pipe_slow);
17560 %}
17561
17562 instruct negD_reg(regD dst) %{
17563 predicate(UseAVX == 0);
17564 match(Set dst (NegD dst));
17565 ins_cost(150);
17566 format %{ "xorpd $dst, [0x8000000000000000]\t"
17567 "# neg double by sign flipping" %}
17568 ins_encode %{
17569 __ xorpd($dst$$XMMRegister, ExternalAddress(double_signflip()));
17570 %}
17571 ins_pipe(pipe_slow);
17572 %}
17573
17574 instruct negD_reg_reg(vlRegD dst, vlRegD src) %{
17575 predicate(UseAVX > 0);
17576 match(Set dst (NegD src));
17577 ins_cost(150);
17578 format %{ "vnegatesd $dst, $src, [0x8000000000000000]\t"
17579 "# neg double by sign flipping" %}
17580 ins_encode %{
17581 __ vnegatesd($dst$$XMMRegister, $src$$XMMRegister,
17582 ExternalAddress(double_signflip()));
17583 %}
17584 ins_pipe(pipe_slow);
17585 %}
17586
17587 // sqrtss instruction needs destination register to be pre initialized for best performance
17588 // Therefore only the instruct rule where the input is pre-loaded into dst register is defined below
17589 instruct sqrtF_reg(regF dst) %{
17590 match(Set dst (SqrtF dst));
17591 format %{ "sqrtss $dst, $dst" %}
17592 ins_encode %{
17593 __ sqrtss($dst$$XMMRegister, $dst$$XMMRegister);
17594 %}
17595 ins_pipe(pipe_slow);
17596 %}
17597
17598 // sqrtsd instruction needs destination register to be pre initialized for best performance
17599 // Therefore only the instruct rule where the input is pre-loaded into dst register is defined below
17600 instruct sqrtD_reg(regD dst) %{
17601 match(Set dst (SqrtD dst));
17602 format %{ "sqrtsd $dst, $dst" %}
17603 ins_encode %{
17604 __ sqrtsd($dst$$XMMRegister, $dst$$XMMRegister);
17605 %}
17606 ins_pipe(pipe_slow);
17607 %}
17608
17609 instruct convF2HF_reg_reg(rRegI dst, vlRegF src, vlRegF tmp) %{
17610 effect(TEMP tmp);
17611 match(Set dst (ConvF2HF src));
17612 ins_cost(125);
17613 format %{ "vcvtps2ph $dst,$src \t using $tmp as TEMP"%}
17614 ins_encode %{
17615 __ flt_to_flt16($dst$$Register, $src$$XMMRegister, $tmp$$XMMRegister);
17616 %}
17617 ins_pipe( pipe_slow );
17618 %}
17619
17620 instruct convF2HF_mem_reg(memory mem, regF src, kReg ktmp, rRegI rtmp) %{
17621 predicate((UseAVX > 2) && VM_Version::supports_avx512vl());
17622 effect(TEMP ktmp, TEMP rtmp);
17623 match(Set mem (StoreC mem (ConvF2HF src)));
17624 format %{ "evcvtps2ph $mem,$src \t using $ktmp and $rtmp as TEMP" %}
17625 ins_encode %{
17626 __ movl($rtmp$$Register, 0x1);
17627 __ kmovwl($ktmp$$KRegister, $rtmp$$Register);
17628 __ evcvtps2ph($mem$$Address, $ktmp$$KRegister, $src$$XMMRegister, 0x04, Assembler::AVX_128bit);
17629 %}
17630 ins_pipe( pipe_slow );
17631 %}
17632
17633 instruct vconvF2HF(vec dst, vec src) %{
17634 match(Set dst (VectorCastF2HF src));
17635 format %{ "vector_conv_F2HF $dst $src" %}
17636 ins_encode %{
17637 int vlen_enc = vector_length_encoding(this, $src);
17638 __ vcvtps2ph($dst$$XMMRegister, $src$$XMMRegister, 0x04, vlen_enc);
17639 %}
17640 ins_pipe( pipe_slow );
17641 %}
17642
17643 instruct vconvF2HF_mem_reg(memory mem, vec src) %{
17644 predicate(n->as_StoreVector()->memory_size() >= 16);
17645 match(Set mem (StoreVector mem (VectorCastF2HF src)));
17646 format %{ "vcvtps2ph $mem,$src" %}
17647 ins_encode %{
17648 int vlen_enc = vector_length_encoding(this, $src);
17649 __ vcvtps2ph($mem$$Address, $src$$XMMRegister, 0x04, vlen_enc);
17650 %}
17651 ins_pipe( pipe_slow );
17652 %}
17653
17654 instruct convHF2F_reg_reg(vlRegF dst, rRegI src) %{
17655 match(Set dst (ConvHF2F src));
17656 format %{ "vcvtph2ps $dst,$src" %}
17657 ins_encode %{
17658 __ flt16_to_flt($dst$$XMMRegister, $src$$Register);
17659 %}
17660 ins_pipe( pipe_slow );
17661 %}
17662
17663 instruct vconvHF2F_reg_mem(vec dst, memory mem) %{
17664 match(Set dst (VectorCastHF2F (LoadVector mem)));
17665 format %{ "vcvtph2ps $dst,$mem" %}
17666 ins_encode %{
17667 int vlen_enc = vector_length_encoding(this);
17668 __ vcvtph2ps($dst$$XMMRegister, $mem$$Address, vlen_enc);
17669 %}
17670 ins_pipe( pipe_slow );
17671 %}
17672
17673 instruct vconvHF2F(vec dst, vec src) %{
17674 match(Set dst (VectorCastHF2F src));
17675 ins_cost(125);
17676 format %{ "vector_conv_HF2F $dst,$src" %}
17677 ins_encode %{
17678 int vlen_enc = vector_length_encoding(this);
17679 __ vcvtph2ps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
17680 %}
17681 ins_pipe( pipe_slow );
17682 %}
17683
17684 // ---------------------------------------- VectorReinterpret ------------------------------------
17685 instruct reinterpret_mask(kReg dst) %{
17686 predicate(n->bottom_type()->isa_pvectmask() &&
17687 Matcher::vector_length(n) == Matcher::vector_length(n->in(1))); // dst == src
17688 match(Set dst (VectorReinterpret dst));
17689 ins_cost(125);
17690 format %{ "vector_reinterpret $dst\t!" %}
17691 ins_encode %{
17692 // empty
17693 %}
17694 ins_pipe( pipe_slow );
17695 %}
17696
17697 instruct reinterpret_mask_W2B(kReg dst, kReg src, vec xtmp) %{
17698 predicate(UseAVX > 2 && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
17699 n->bottom_type()->isa_pvectmask() &&
17700 n->in(1)->bottom_type()->isa_pvectmask() &&
17701 n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_SHORT &&
17702 n->bottom_type()->is_pvectmask()->element_basic_type() == T_BYTE); // dst == src
17703 match(Set dst (VectorReinterpret src));
17704 effect(TEMP xtmp);
17705 format %{ "vector_mask_reinterpret_W2B $dst $src\t!" %}
17706 ins_encode %{
17707 int src_sz = Matcher::vector_length(this, $src)*type2aelembytes(T_SHORT);
17708 int dst_sz = Matcher::vector_length(this)*type2aelembytes(T_BYTE);
17709 assert(src_sz == dst_sz , "src and dst size mismatch");
17710 int vlen_enc = vector_length_encoding(src_sz);
17711 __ evpmovm2w($xtmp$$XMMRegister, $src$$KRegister, vlen_enc);
17712 __ evpmovb2m($dst$$KRegister, $xtmp$$XMMRegister, vlen_enc);
17713 %}
17714 ins_pipe( pipe_slow );
17715 %}
17716
17717 instruct reinterpret_mask_D2B(kReg dst, kReg src, vec xtmp) %{
17718 predicate(UseAVX > 2 && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
17719 n->bottom_type()->isa_pvectmask() &&
17720 n->in(1)->bottom_type()->isa_pvectmask() &&
17721 (n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_INT ||
17722 n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_FLOAT) &&
17723 n->bottom_type()->is_pvectmask()->element_basic_type() == T_BYTE); // dst == src
17724 match(Set dst (VectorReinterpret src));
17725 effect(TEMP xtmp);
17726 format %{ "vector_mask_reinterpret_D2B $dst $src\t!" %}
17727 ins_encode %{
17728 int src_sz = Matcher::vector_length(this, $src)*type2aelembytes(T_INT);
17729 int dst_sz = Matcher::vector_length(this)*type2aelembytes(T_BYTE);
17730 assert(src_sz == dst_sz , "src and dst size mismatch");
17731 int vlen_enc = vector_length_encoding(src_sz);
17732 __ evpmovm2d($xtmp$$XMMRegister, $src$$KRegister, vlen_enc);
17733 __ evpmovb2m($dst$$KRegister, $xtmp$$XMMRegister, vlen_enc);
17734 %}
17735 ins_pipe( pipe_slow );
17736 %}
17737
17738 instruct reinterpret_mask_Q2B(kReg dst, kReg src, vec xtmp) %{
17739 predicate(UseAVX > 2 && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
17740 n->bottom_type()->isa_pvectmask() &&
17741 n->in(1)->bottom_type()->isa_pvectmask() &&
17742 (n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_LONG ||
17743 n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_DOUBLE) &&
17744 n->bottom_type()->is_pvectmask()->element_basic_type() == T_BYTE); // dst == src
17745 match(Set dst (VectorReinterpret src));
17746 effect(TEMP xtmp);
17747 format %{ "vector_mask_reinterpret_Q2B $dst $src\t!" %}
17748 ins_encode %{
17749 int src_sz = Matcher::vector_length(this, $src)*type2aelembytes(T_LONG);
17750 int dst_sz = Matcher::vector_length(this)*type2aelembytes(T_BYTE);
17751 assert(src_sz == dst_sz , "src and dst size mismatch");
17752 int vlen_enc = vector_length_encoding(src_sz);
17753 __ evpmovm2q($xtmp$$XMMRegister, $src$$KRegister, vlen_enc);
17754 __ evpmovb2m($dst$$KRegister, $xtmp$$XMMRegister, vlen_enc);
17755 %}
17756 ins_pipe( pipe_slow );
17757 %}
17758
17759 instruct reinterpret(vec dst) %{
17760 predicate(!n->bottom_type()->isa_pvectmask() &&
17761 Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1))); // dst == src
17762 match(Set dst (VectorReinterpret dst));
17763 ins_cost(125);
17764 format %{ "vector_reinterpret $dst\t!" %}
17765 ins_encode %{
17766 // empty
17767 %}
17768 ins_pipe( pipe_slow );
17769 %}
17770
17771 instruct reinterpret_expand(vec dst, vec src) %{
17772 predicate(UseAVX == 0 &&
17773 (Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
17774 match(Set dst (VectorReinterpret src));
17775 ins_cost(125);
17776 effect(TEMP dst);
17777 format %{ "vector_reinterpret_expand $dst,$src" %}
17778 ins_encode %{
17779 assert(Matcher::vector_length_in_bytes(this) <= 16, "required");
17780 assert(Matcher::vector_length_in_bytes(this, $src) <= 8, "required");
17781
17782 int src_vlen_in_bytes = Matcher::vector_length_in_bytes(this, $src);
17783 if (src_vlen_in_bytes == 4) {
17784 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_32_bit_mask()), noreg);
17785 } else {
17786 assert(src_vlen_in_bytes == 8, "");
17787 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_64_bit_mask()), noreg);
17788 }
17789 __ pand($dst$$XMMRegister, $src$$XMMRegister);
17790 %}
17791 ins_pipe( pipe_slow );
17792 %}
17793
17794 instruct vreinterpret_expand4(legVec dst, vec src) %{
17795 predicate(UseAVX > 0 &&
17796 !n->bottom_type()->isa_pvectmask() &&
17797 (Matcher::vector_length_in_bytes(n->in(1)) == 4) && // src
17798 (Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
17799 match(Set dst (VectorReinterpret src));
17800 ins_cost(125);
17801 format %{ "vector_reinterpret_expand $dst,$src" %}
17802 ins_encode %{
17803 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_32_bit_mask()), 0, noreg);
17804 %}
17805 ins_pipe( pipe_slow );
17806 %}
17807
17808
17809 instruct vreinterpret_expand(legVec dst, vec src) %{
17810 predicate(UseAVX > 0 &&
17811 !n->bottom_type()->isa_pvectmask() &&
17812 (Matcher::vector_length_in_bytes(n->in(1)) > 4) && // src
17813 (Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
17814 match(Set dst (VectorReinterpret src));
17815 ins_cost(125);
17816 format %{ "vector_reinterpret_expand $dst,$src\t!" %}
17817 ins_encode %{
17818 switch (Matcher::vector_length_in_bytes(this, $src)) {
17819 case 8: __ movq ($dst$$XMMRegister, $src$$XMMRegister); break;
17820 case 16: __ movdqu ($dst$$XMMRegister, $src$$XMMRegister); break;
17821 case 32: __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister); break;
17822 default: ShouldNotReachHere();
17823 }
17824 %}
17825 ins_pipe( pipe_slow );
17826 %}
17827
17828 instruct reinterpret_shrink(vec dst, legVec src) %{
17829 predicate(!n->bottom_type()->isa_pvectmask() &&
17830 Matcher::vector_length_in_bytes(n->in(1)) > Matcher::vector_length_in_bytes(n)); // src > dst
17831 match(Set dst (VectorReinterpret src));
17832 ins_cost(125);
17833 format %{ "vector_reinterpret_shrink $dst,$src\t!" %}
17834 ins_encode %{
17835 switch (Matcher::vector_length_in_bytes(this)) {
17836 case 4: __ movfltz($dst$$XMMRegister, $src$$XMMRegister); break;
17837 case 8: __ movq ($dst$$XMMRegister, $src$$XMMRegister); break;
17838 case 16: __ movdqu ($dst$$XMMRegister, $src$$XMMRegister); break;
17839 case 32: __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister); break;
17840 default: ShouldNotReachHere();
17841 }
17842 %}
17843 ins_pipe( pipe_slow );
17844 %}
17845
17846 // ----------------------------------------------------------------------------------------------------
17847
17848 instruct roundD_reg(legRegD dst, legRegD src, immU8 rmode) %{
17849 match(Set dst (RoundDoubleMode src rmode));
17850 format %{ "roundsd $dst,$src" %}
17851 ins_cost(150);
17852 ins_encode %{
17853 assert(UseSSE >= 4, "required");
17854 if ((UseAVX == 0) && ($dst$$XMMRegister != $src$$XMMRegister)) {
17855 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
17856 }
17857 __ roundsd($dst$$XMMRegister, $src$$XMMRegister, $rmode$$constant);
17858 %}
17859 ins_pipe(pipe_slow);
17860 %}
17861
17862 instruct roundD_imm(legRegD dst, immD con, immU8 rmode) %{
17863 match(Set dst (RoundDoubleMode con rmode));
17864 format %{ "roundsd $dst,[$constantaddress]\t# load from constant table: double=$con" %}
17865 ins_cost(150);
17866 ins_encode %{
17867 assert(UseSSE >= 4, "required");
17868 __ roundsd($dst$$XMMRegister, $constantaddress($con), $rmode$$constant, noreg);
17869 %}
17870 ins_pipe(pipe_slow);
17871 %}
17872
17873 instruct vroundD_reg(legVec dst, legVec src, immU8 rmode) %{
17874 predicate(Matcher::vector_length(n) < 8);
17875 match(Set dst (RoundDoubleModeV src rmode));
17876 format %{ "vroundpd $dst,$src,$rmode\t! round packedD" %}
17877 ins_encode %{
17878 assert(UseAVX > 0, "required");
17879 int vlen_enc = vector_length_encoding(this);
17880 __ vroundpd($dst$$XMMRegister, $src$$XMMRegister, $rmode$$constant, vlen_enc);
17881 %}
17882 ins_pipe( pipe_slow );
17883 %}
17884
17885 instruct vround8D_reg(vec dst, vec src, immU8 rmode) %{
17886 predicate(Matcher::vector_length(n) == 8);
17887 match(Set dst (RoundDoubleModeV src rmode));
17888 format %{ "vrndscalepd $dst,$src,$rmode\t! round packed8D" %}
17889 ins_encode %{
17890 assert(UseAVX > 2, "required");
17891 __ vrndscalepd($dst$$XMMRegister, $src$$XMMRegister, $rmode$$constant, Assembler::AVX_512bit);
17892 %}
17893 ins_pipe( pipe_slow );
17894 %}
17895
17896 instruct vroundD_mem(legVec dst, memory mem, immU8 rmode) %{
17897 predicate(Matcher::vector_length(n) < 8);
17898 match(Set dst (RoundDoubleModeV (LoadVector mem) rmode));
17899 format %{ "vroundpd $dst, $mem, $rmode\t! round packedD" %}
17900 ins_encode %{
17901 assert(UseAVX > 0, "required");
17902 int vlen_enc = vector_length_encoding(this);
17903 __ vroundpd($dst$$XMMRegister, $mem$$Address, $rmode$$constant, vlen_enc);
17904 %}
17905 ins_pipe( pipe_slow );
17906 %}
17907
17908 instruct vround8D_mem(vec dst, memory mem, immU8 rmode) %{
17909 predicate(Matcher::vector_length(n) == 8);
17910 match(Set dst (RoundDoubleModeV (LoadVector mem) rmode));
17911 format %{ "vrndscalepd $dst,$mem,$rmode\t! round packed8D" %}
17912 ins_encode %{
17913 assert(UseAVX > 2, "required");
17914 __ vrndscalepd($dst$$XMMRegister, $mem$$Address, $rmode$$constant, Assembler::AVX_512bit);
17915 %}
17916 ins_pipe( pipe_slow );
17917 %}
17918
17919 instruct onspinwait() %{
17920 match(OnSpinWait);
17921 ins_cost(200);
17922
17923 format %{
17924 $$template
17925 $$emit$$"pause\t! membar_onspinwait"
17926 %}
17927 ins_encode %{
17928 __ pause();
17929 %}
17930 ins_pipe(pipe_slow);
17931 %}
17932
17933 // a * b + c
17934 instruct fmaD_reg(regD a, regD b, regD c) %{
17935 match(Set c (FmaD c (Binary a b)));
17936 format %{ "fmasd $a,$b,$c\t# $c = $a * $b + $c" %}
17937 ins_cost(150);
17938 ins_encode %{
17939 assert(UseFMA, "Needs FMA instructions support.");
17940 __ fmad($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister);
17941 %}
17942 ins_pipe( pipe_slow );
17943 %}
17944
17945 // a * b + c
17946 instruct fmaF_reg(regF a, regF b, regF c) %{
17947 match(Set c (FmaF c (Binary a b)));
17948 format %{ "fmass $a,$b,$c\t# $c = $a * $b + $c" %}
17949 ins_cost(150);
17950 ins_encode %{
17951 assert(UseFMA, "Needs FMA instructions support.");
17952 __ fmaf($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister);
17953 %}
17954 ins_pipe( pipe_slow );
17955 %}
17956
17957 // ====================VECTOR INSTRUCTIONS=====================================
17958
17959 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
17960 instruct MoveVec2Leg(legVec dst, vec src) %{
17961 match(Set dst src);
17962 format %{ "" %}
17963 ins_encode %{
17964 ShouldNotReachHere();
17965 %}
17966 ins_pipe( fpu_reg_reg );
17967 %}
17968
17969 instruct MoveLeg2Vec(vec dst, legVec src) %{
17970 match(Set dst src);
17971 format %{ "" %}
17972 ins_encode %{
17973 ShouldNotReachHere();
17974 %}
17975 ins_pipe( fpu_reg_reg );
17976 %}
17977
17978 // ============================================================================
17979
17980 // Load vectors generic operand pattern
17981 instruct loadV(vec dst, memory mem) %{
17982 match(Set dst (LoadVector mem));
17983 ins_cost(125);
17984 format %{ "load_vector $dst,$mem" %}
17985 ins_encode %{
17986 BasicType bt = Matcher::vector_element_basic_type(this);
17987 __ load_vector(bt, $dst$$XMMRegister, $mem$$Address, Matcher::vector_length_in_bytes(this));
17988 %}
17989 ins_pipe( pipe_slow );
17990 %}
17991
17992 // Store vectors generic operand pattern.
17993 instruct storeV(memory mem, vec src) %{
17994 match(Set mem (StoreVector mem src));
17995 ins_cost(145);
17996 format %{ "store_vector $mem,$src\n\t" %}
17997 ins_encode %{
17998 switch (Matcher::vector_length_in_bytes(this, $src)) {
17999 case 4: __ movdl ($mem$$Address, $src$$XMMRegister); break;
18000 case 8: __ movq ($mem$$Address, $src$$XMMRegister); break;
18001 case 16: __ movdqu ($mem$$Address, $src$$XMMRegister); break;
18002 case 32: __ vmovdqu ($mem$$Address, $src$$XMMRegister); break;
18003 case 64: __ evmovdqul($mem$$Address, $src$$XMMRegister, Assembler::AVX_512bit); break;
18004 default: ShouldNotReachHere();
18005 }
18006 %}
18007 ins_pipe( pipe_slow );
18008 %}
18009
18010 // ---------------------------------------- Gather ------------------------------------
18011
18012 // Gather BYTE, SHORT, INT, LONG, FLOAT, DOUBLE
18013
18014 instruct gather(legVec dst, memory mem, legVec idx, rRegP tmp, legVec mask) %{
18015 predicate(!VM_Version::supports_avx512vl() && !is_subword_type(Matcher::vector_element_basic_type(n)) &&
18016 Matcher::vector_length_in_bytes(n) <= 32);
18017 match(Set dst (LoadVectorGather mem idx));
18018 effect(TEMP dst, TEMP tmp, TEMP mask);
18019 format %{ "load_vector_gather $dst, $mem, $idx\t! using $tmp and $mask as TEMP" %}
18020 ins_encode %{
18021 int vlen_enc = vector_length_encoding(this);
18022 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18023 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18024 __ vpcmpeqd($mask$$XMMRegister, $mask$$XMMRegister, $mask$$XMMRegister, vlen_enc);
18025 __ lea($tmp$$Register, $mem$$Address);
18026 __ vgather(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx$$XMMRegister, $mask$$XMMRegister, vlen_enc);
18027 %}
18028 ins_pipe( pipe_slow );
18029 %}
18030
18031
18032 instruct evgather(vec dst, memory mem, vec idx, rRegP tmp, kReg ktmp) %{
18033 predicate((VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64) &&
18034 !is_subword_type(Matcher::vector_element_basic_type(n)));
18035 match(Set dst (LoadVectorGather mem idx));
18036 effect(TEMP dst, TEMP tmp, TEMP ktmp);
18037 format %{ "load_vector_gather $dst, $mem, $idx\t! using $tmp and ktmp as TEMP" %}
18038 ins_encode %{
18039 int vlen_enc = vector_length_encoding(this);
18040 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18041 __ kxnorwl($ktmp$$KRegister, $ktmp$$KRegister, $ktmp$$KRegister);
18042 __ lea($tmp$$Register, $mem$$Address);
18043 __ evgather(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $tmp$$Register, $idx$$XMMRegister, vlen_enc);
18044 %}
18045 ins_pipe( pipe_slow );
18046 %}
18047
18048 instruct evgather_masked(vec dst, memory mem, vec idx, kReg mask, kReg ktmp, rRegP tmp) %{
18049 predicate((VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64) &&
18050 !is_subword_type(Matcher::vector_element_basic_type(n)));
18051 match(Set dst (LoadVectorGatherMasked mem (Binary idx mask)));
18052 effect(TEMP_DEF dst, TEMP tmp, TEMP ktmp);
18053 format %{ "load_vector_gather_masked $dst, $mem, $idx, $mask\t! using $tmp and ktmp as TEMP" %}
18054 ins_encode %{
18055 assert(UseAVX > 2, "sanity");
18056 int vlen_enc = vector_length_encoding(this);
18057 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18058 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18059 // Note: Since gather instruction partially updates the opmask register used
18060 // for predication hense moving mask operand to a temporary.
18061 __ kmovwl($ktmp$$KRegister, $mask$$KRegister);
18062 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18063 __ lea($tmp$$Register, $mem$$Address);
18064 __ evgather(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $tmp$$Register, $idx$$XMMRegister, vlen_enc);
18065 %}
18066 ins_pipe( pipe_slow );
18067 %}
18068
18069 instruct vgather_subwordLE8B(vec dst, memory mem, rRegP idx_base, rRegP tmp, rRegI rtmp) %{
18070 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) <= 8);
18071 match(Set dst (LoadVectorGather mem idx_base));
18072 effect(TEMP tmp, TEMP rtmp);
18073 format %{ "vector_gatherLE8 $dst, $mem, $idx_base\t! using $tmp and $rtmp as TEMP" %}
18074 ins_encode %{
18075 int vlen_enc = vector_length_encoding(this);
18076 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18077 __ lea($tmp$$Register, $mem$$Address);
18078 __ vgather8b(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base$$Register, $rtmp$$Register, vlen_enc);
18079 %}
18080 ins_pipe( pipe_slow );
18081 %}
18082
18083 instruct vgather_subwordGT8B(vec dst, memory mem, rRegP idx_base, rRegP tmp, rRegP idx_base_temp,
18084 vec xtmp1, vec xtmp2, vec xtmp3, rRegI rtmp, rRegI length, rFlagsReg cr) %{
18085 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) > 8);
18086 match(Set dst (LoadVectorGather mem idx_base));
18087 effect(TEMP_DEF dst, TEMP tmp, TEMP idx_base_temp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp, TEMP length, KILL cr);
18088 format %{ "vector_gatherGT8 $dst, $mem, $idx_base\t! using $tmp, $idx_base_temp, $xtmp1, $xtmp2, $xtmp3, $rtmp and $length as TEMP" %}
18089 ins_encode %{
18090 int vlen_enc = vector_length_encoding(this);
18091 int vector_len = Matcher::vector_length(this);
18092 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18093 __ lea($tmp$$Register, $mem$$Address);
18094 __ movptr($idx_base_temp$$Register, $idx_base$$Register);
18095 __ vgather_subword(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base_temp$$Register, noreg, $xtmp1$$XMMRegister,
18096 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, noreg, $length$$Register, vector_len, vlen_enc);
18097 %}
18098 ins_pipe( pipe_slow );
18099 %}
18100
18101 instruct vgather_masked_subwordLE8B_avx3(vec dst, memory mem, rRegP idx_base, kReg mask, rRegL mask_idx, rRegP tmp, rRegI rtmp, rRegL rtmp2, rFlagsReg cr) %{
18102 predicate(VM_Version::supports_avx512bw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) <= 8);
18103 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18104 effect(TEMP mask_idx, TEMP tmp, TEMP rtmp, TEMP rtmp2, KILL cr);
18105 format %{ "vector_masked_gatherLE8 $dst, $mem, $idx_base, $mask\t! using $mask_idx, $tmp, $rtmp and $rtmp2 as TEMP" %}
18106 ins_encode %{
18107 int vlen_enc = vector_length_encoding(this);
18108 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18109 __ xorq($mask_idx$$Register, $mask_idx$$Register);
18110 __ lea($tmp$$Register, $mem$$Address);
18111 __ kmovql($rtmp2$$Register, $mask$$KRegister);
18112 __ vgather8b_masked(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base$$Register, $rtmp2$$Register, $mask_idx$$Register, $rtmp$$Register, vlen_enc);
18113 %}
18114 ins_pipe( pipe_slow );
18115 %}
18116
18117 instruct vgather_masked_subwordGT8B_avx3(vec dst, memory mem, rRegP idx_base, kReg mask, rRegP tmp, rRegP idx_base_temp,
18118 vec xtmp1, vec xtmp2, vec xtmp3, rRegI rtmp, rRegL rtmp2, rRegL mask_idx, rRegI length, rFlagsReg cr) %{
18119 predicate(VM_Version::supports_avx512bw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) > 8);
18120 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18121 effect(TEMP_DEF dst, TEMP tmp, TEMP idx_base_temp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp, TEMP rtmp2, TEMP mask_idx, TEMP length, KILL cr);
18122 format %{ "vector_gatherGT8_masked $dst, $mem, $idx_base, $mask\t! using $tmp, $idx_base_temp, $xtmp1, $xtmp2, $xtmp3, $rtmp, $rtmp2, $mask_idx and $length as TEMP" %}
18123 ins_encode %{
18124 int vlen_enc = vector_length_encoding(this);
18125 int vector_len = Matcher::vector_length(this);
18126 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18127 __ xorq($mask_idx$$Register, $mask_idx$$Register);
18128 __ lea($tmp$$Register, $mem$$Address);
18129 __ movptr($idx_base_temp$$Register, $idx_base$$Register);
18130 __ kmovql($rtmp2$$Register, $mask$$KRegister);
18131 __ vgather_subword(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base_temp$$Register, $rtmp2$$Register, $xtmp1$$XMMRegister,
18132 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, $mask_idx$$Register, $length$$Register, vector_len, vlen_enc);
18133 %}
18134 ins_pipe( pipe_slow );
18135 %}
18136
18137 instruct vgather_masked_subwordLE8B_avx2(vec dst, memory mem, rRegP idx_base, vec mask, rRegI mask_idx, rRegP tmp, rRegI rtmp, rRegI rtmp2, rFlagsReg cr) %{
18138 predicate(!VM_Version::supports_avx512vlbw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) <= 8);
18139 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18140 effect(TEMP mask_idx, TEMP tmp, TEMP rtmp, TEMP rtmp2, KILL cr);
18141 format %{ "vector_masked_gatherLE8 $dst, $mem, $idx_base, $mask\t! using $mask_idx, $tmp, $rtmp and $rtmp2 as TEMP" %}
18142 ins_encode %{
18143 int vlen_enc = vector_length_encoding(this);
18144 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18145 __ lea($tmp$$Register, $mem$$Address);
18146 __ vpmovmskb($rtmp2$$Register, $mask$$XMMRegister, vlen_enc);
18147 if (elem_bt == T_SHORT) {
18148 __ movl($mask_idx$$Register, 0x55555555);
18149 __ pextl($rtmp2$$Register, $rtmp2$$Register, $mask_idx$$Register);
18150 }
18151 __ xorl($mask_idx$$Register, $mask_idx$$Register);
18152 __ vgather8b_masked(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base$$Register, $rtmp2$$Register, $mask_idx$$Register, $rtmp$$Register, vlen_enc);
18153 %}
18154 ins_pipe( pipe_slow );
18155 %}
18156
18157 instruct vgather_masked_subwordGT8B_avx2(vec dst, memory mem, rRegP idx_base, vec mask, rRegP tmp, rRegP idx_base_temp,
18158 vec xtmp1, vec xtmp2, vec xtmp3, rRegI rtmp, rRegI rtmp2, rRegI mask_idx, rRegI length, rFlagsReg cr) %{
18159 predicate(!VM_Version::supports_avx512vlbw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) > 8);
18160 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18161 effect(TEMP_DEF dst, TEMP tmp, TEMP idx_base_temp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp, TEMP rtmp2, TEMP mask_idx, TEMP length, KILL cr);
18162 format %{ "vector_gatherGT8_masked $dst, $mem, $idx_base, $mask\t! using $tmp, $idx_base_temp, $xtmp1, $xtmp2, $xtmp3, $rtmp, $rtmp2, $mask_idx and $length as TEMP" %}
18163 ins_encode %{
18164 int vlen_enc = vector_length_encoding(this);
18165 int vector_len = Matcher::vector_length(this);
18166 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18167 __ lea($tmp$$Register, $mem$$Address);
18168 __ movptr($idx_base_temp$$Register, $idx_base$$Register);
18169 __ vpmovmskb($rtmp2$$Register, $mask$$XMMRegister, vlen_enc);
18170 if (elem_bt == T_SHORT) {
18171 __ movl($mask_idx$$Register, 0x55555555);
18172 __ pextl($rtmp2$$Register, $rtmp2$$Register, $mask_idx$$Register);
18173 }
18174 __ xorl($mask_idx$$Register, $mask_idx$$Register);
18175 __ vgather_subword(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base_temp$$Register, $rtmp2$$Register, $xtmp1$$XMMRegister,
18176 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, $mask_idx$$Register, $length$$Register, vector_len, vlen_enc);
18177 %}
18178 ins_pipe( pipe_slow );
18179 %}
18180
18181 // ====================Scatter=======================================
18182
18183 // Scatter INT, LONG, FLOAT, DOUBLE
18184
18185 instruct scatter(memory mem, vec src, vec idx, rRegP tmp, kReg ktmp) %{
18186 predicate(UseAVX > 2);
18187 match(Set mem (StoreVectorScatter mem (Binary src idx)));
18188 effect(TEMP tmp, TEMP ktmp);
18189 format %{ "store_vector_scatter $mem, $idx, $src\t! using k2 and $tmp as TEMP" %}
18190 ins_encode %{
18191 int vlen_enc = vector_length_encoding(this, $src);
18192 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
18193
18194 assert(Matcher::vector_length_in_bytes(this, $src) >= 16, "sanity");
18195 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18196
18197 __ kmovwl($ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), noreg);
18198 __ lea($tmp$$Register, $mem$$Address);
18199 __ evscatter(elem_bt, $tmp$$Register, $idx$$XMMRegister, $ktmp$$KRegister, $src$$XMMRegister, vlen_enc);
18200 %}
18201 ins_pipe( pipe_slow );
18202 %}
18203
18204 instruct scatter_masked(memory mem, vec src, vec idx, kReg mask, kReg ktmp, rRegP tmp) %{
18205 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx mask))));
18206 effect(TEMP tmp, TEMP ktmp);
18207 format %{ "store_vector_scatter_masked $mem, $idx, $src, $mask\t!" %}
18208 ins_encode %{
18209 int vlen_enc = vector_length_encoding(this, $src);
18210 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
18211 assert(Matcher::vector_length_in_bytes(this, $src) >= 16, "sanity");
18212 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18213 // Note: Since scatter instruction partially updates the opmask register used
18214 // for predication hense moving mask operand to a temporary.
18215 __ kmovwl($ktmp$$KRegister, $mask$$KRegister);
18216 __ lea($tmp$$Register, $mem$$Address);
18217 __ evscatter(elem_bt, $tmp$$Register, $idx$$XMMRegister, $ktmp$$KRegister, $src$$XMMRegister, vlen_enc);
18218 %}
18219 ins_pipe( pipe_slow );
18220 %}
18221
18222 // ====================REPLICATE=======================================
18223
18224 // Replicate byte scalar to be vector
18225 instruct vReplB_reg(vec dst, rRegI src) %{
18226 predicate(Matcher::vector_element_basic_type(n) == T_BYTE);
18227 match(Set dst (Replicate src));
18228 format %{ "replicateB $dst,$src" %}
18229 ins_encode %{
18230 uint vlen = Matcher::vector_length(this);
18231 if (UseAVX >= 2) {
18232 int vlen_enc = vector_length_encoding(this);
18233 if (vlen == 64 || VM_Version::supports_avx512vlbw()) { // AVX512VL for <512bit operands
18234 assert(VM_Version::supports_avx512bw(), "required"); // 512-bit byte vectors assume AVX512BW
18235 __ evpbroadcastb($dst$$XMMRegister, $src$$Register, vlen_enc);
18236 } else {
18237 __ movdl($dst$$XMMRegister, $src$$Register);
18238 __ vpbroadcastb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18239 }
18240 } else {
18241 assert(UseAVX < 2, "");
18242 __ movdl($dst$$XMMRegister, $src$$Register);
18243 __ punpcklbw($dst$$XMMRegister, $dst$$XMMRegister);
18244 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18245 if (vlen >= 16) {
18246 assert(vlen == 16, "");
18247 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18248 }
18249 }
18250 %}
18251 ins_pipe( pipe_slow );
18252 %}
18253
18254 instruct ReplB_mem(vec dst, memory mem) %{
18255 predicate(UseAVX >= 2 && Matcher::vector_element_basic_type(n) == T_BYTE);
18256 match(Set dst (Replicate (LoadB mem)));
18257 format %{ "replicateB $dst,$mem" %}
18258 ins_encode %{
18259 int vlen_enc = vector_length_encoding(this);
18260 __ vpbroadcastb($dst$$XMMRegister, $mem$$Address, vlen_enc);
18261 %}
18262 ins_pipe( pipe_slow );
18263 %}
18264
18265 // ====================ReplicateS=======================================
18266
18267 instruct vReplS_reg(vec dst, rRegI src) %{
18268 predicate(Matcher::vector_element_basic_type(n) == T_SHORT);
18269 match(Set dst (Replicate src));
18270 format %{ "replicateS $dst,$src" %}
18271 ins_encode %{
18272 uint vlen = Matcher::vector_length(this);
18273 int vlen_enc = vector_length_encoding(this);
18274 if (UseAVX >= 2) {
18275 if (vlen == 32 || VM_Version::supports_avx512vlbw()) { // AVX512VL for <512bit operands
18276 assert(VM_Version::supports_avx512bw(), "required"); // 512-bit short vectors assume AVX512BW
18277 __ evpbroadcastw($dst$$XMMRegister, $src$$Register, vlen_enc);
18278 } else {
18279 __ movdl($dst$$XMMRegister, $src$$Register);
18280 __ vpbroadcastw($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18281 }
18282 } else {
18283 assert(UseAVX < 2, "");
18284 __ movdl($dst$$XMMRegister, $src$$Register);
18285 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18286 if (vlen >= 8) {
18287 assert(vlen == 8, "");
18288 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18289 }
18290 }
18291 %}
18292 ins_pipe( pipe_slow );
18293 %}
18294
18295 instruct ReplHF_imm(vec dst, immH con, rRegI rtmp) %{
18296 match(Set dst (Replicate con));
18297 effect(TEMP rtmp);
18298 format %{ "replicateHF $dst, $con \t! using $rtmp as TEMP" %}
18299 ins_encode %{
18300 int vlen_enc = vector_length_encoding(this);
18301 BasicType bt = Matcher::vector_element_basic_type(this);
18302 assert(VM_Version::supports_avx512_fp16() && bt == T_SHORT, "");
18303 __ movl($rtmp$$Register, $con$$constant);
18304 __ evpbroadcastw($dst$$XMMRegister, $rtmp$$Register, vlen_enc);
18305 %}
18306 ins_pipe( pipe_slow );
18307 %}
18308
18309 instruct ReplHF_reg(vec dst, regF src, rRegI rtmp) %{
18310 predicate(VM_Version::supports_avx512_fp16() && Matcher::vector_element_basic_type(n) == T_SHORT);
18311 match(Set dst (Replicate src));
18312 effect(TEMP rtmp);
18313 format %{ "replicateHF $dst, $src \t! using $rtmp as TEMP" %}
18314 ins_encode %{
18315 int vlen_enc = vector_length_encoding(this);
18316 __ evmovw($rtmp$$Register, $src$$XMMRegister);
18317 __ evpbroadcastw($dst$$XMMRegister, $rtmp$$Register, vlen_enc);
18318 %}
18319 ins_pipe( pipe_slow );
18320 %}
18321
18322 instruct ReplS_mem(vec dst, memory mem) %{
18323 predicate(UseAVX >= 2 && Matcher::vector_element_basic_type(n) == T_SHORT);
18324 match(Set dst (Replicate (LoadS mem)));
18325 format %{ "replicateS $dst,$mem" %}
18326 ins_encode %{
18327 int vlen_enc = vector_length_encoding(this);
18328 __ vpbroadcastw($dst$$XMMRegister, $mem$$Address, vlen_enc);
18329 %}
18330 ins_pipe( pipe_slow );
18331 %}
18332
18333 // ====================ReplicateI=======================================
18334
18335 instruct ReplI_reg(vec dst, rRegI src) %{
18336 predicate(Matcher::vector_element_basic_type(n) == T_INT);
18337 match(Set dst (Replicate src));
18338 format %{ "replicateI $dst,$src" %}
18339 ins_encode %{
18340 uint vlen = Matcher::vector_length(this);
18341 int vlen_enc = vector_length_encoding(this);
18342 if (vlen == 16 || VM_Version::supports_avx512vl()) { // AVX512VL for <512bit operands
18343 __ evpbroadcastd($dst$$XMMRegister, $src$$Register, vlen_enc);
18344 } else if (VM_Version::supports_avx2()) {
18345 __ movdl($dst$$XMMRegister, $src$$Register);
18346 __ vpbroadcastd($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18347 } else {
18348 __ movdl($dst$$XMMRegister, $src$$Register);
18349 __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18350 }
18351 %}
18352 ins_pipe( pipe_slow );
18353 %}
18354
18355 instruct ReplI_mem(vec dst, memory mem) %{
18356 predicate(Matcher::vector_element_basic_type(n) == T_INT);
18357 match(Set dst (Replicate (LoadI mem)));
18358 format %{ "replicateI $dst,$mem" %}
18359 ins_encode %{
18360 int vlen_enc = vector_length_encoding(this);
18361 if (VM_Version::supports_avx2()) {
18362 __ vpbroadcastd($dst$$XMMRegister, $mem$$Address, vlen_enc);
18363 } else if (VM_Version::supports_avx()) {
18364 __ vbroadcastss($dst$$XMMRegister, $mem$$Address, vlen_enc);
18365 } else {
18366 __ movdl($dst$$XMMRegister, $mem$$Address);
18367 __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18368 }
18369 %}
18370 ins_pipe( pipe_slow );
18371 %}
18372
18373 instruct ReplI_imm(vec dst, immI con) %{
18374 predicate(Matcher::is_non_long_integral_vector(n));
18375 match(Set dst (Replicate con));
18376 format %{ "replicateI $dst,$con" %}
18377 ins_encode %{
18378 InternalAddress addr = $constantaddress(vreplicate_imm(Matcher::vector_element_basic_type(this), $con$$constant,
18379 (VM_Version::supports_sse3() ? (VM_Version::supports_avx() ? 4 : 8) : 16) /
18380 type2aelembytes(Matcher::vector_element_basic_type(this))));
18381 BasicType bt = Matcher::vector_element_basic_type(this);
18382 int vlen = Matcher::vector_length_in_bytes(this);
18383 __ load_constant_vector(bt, $dst$$XMMRegister, addr, vlen);
18384 %}
18385 ins_pipe( pipe_slow );
18386 %}
18387
18388 // Replicate scalar zero to be vector
18389 instruct ReplI_zero(vec dst, immI_0 zero) %{
18390 predicate(Matcher::is_non_long_integral_vector(n));
18391 match(Set dst (Replicate zero));
18392 format %{ "replicateI $dst,$zero" %}
18393 ins_encode %{
18394 int vlen_enc = vector_length_encoding(this);
18395 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vl()) {
18396 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18397 } else {
18398 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
18399 }
18400 %}
18401 ins_pipe( fpu_reg_reg );
18402 %}
18403
18404 instruct ReplI_M1(vec dst, immI_M1 con) %{
18405 predicate(Matcher::is_non_long_integral_vector(n));
18406 match(Set dst (Replicate con));
18407 format %{ "vallones $dst" %}
18408 ins_encode %{
18409 int vector_len = vector_length_encoding(this);
18410 __ vallones($dst$$XMMRegister, vector_len);
18411 %}
18412 ins_pipe( pipe_slow );
18413 %}
18414
18415 // ====================ReplicateL=======================================
18416
18417 // Replicate long (8 byte) scalar to be vector
18418 instruct ReplL_reg(vec dst, rRegL src) %{
18419 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18420 match(Set dst (Replicate src));
18421 format %{ "replicateL $dst,$src" %}
18422 ins_encode %{
18423 int vlen = Matcher::vector_length(this);
18424 int vlen_enc = vector_length_encoding(this);
18425 if (vlen == 8 || VM_Version::supports_avx512vl()) { // AVX512VL for <512bit operands
18426 __ evpbroadcastq($dst$$XMMRegister, $src$$Register, vlen_enc);
18427 } else if (VM_Version::supports_avx2()) {
18428 __ movdq($dst$$XMMRegister, $src$$Register);
18429 __ vpbroadcastq($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18430 } else {
18431 __ movdq($dst$$XMMRegister, $src$$Register);
18432 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18433 }
18434 %}
18435 ins_pipe( pipe_slow );
18436 %}
18437
18438 instruct ReplL_mem(vec dst, memory mem) %{
18439 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18440 match(Set dst (Replicate (LoadL mem)));
18441 format %{ "replicateL $dst,$mem" %}
18442 ins_encode %{
18443 int vlen_enc = vector_length_encoding(this);
18444 if (VM_Version::supports_avx2()) {
18445 __ vpbroadcastq($dst$$XMMRegister, $mem$$Address, vlen_enc);
18446 } else if (VM_Version::supports_sse3()) {
18447 __ movddup($dst$$XMMRegister, $mem$$Address);
18448 } else {
18449 __ movq($dst$$XMMRegister, $mem$$Address);
18450 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18451 }
18452 %}
18453 ins_pipe( pipe_slow );
18454 %}
18455
18456 // Replicate long (8 byte) scalar immediate to be vector by loading from const table.
18457 instruct ReplL_imm(vec dst, immL con) %{
18458 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18459 match(Set dst (Replicate con));
18460 format %{ "replicateL $dst,$con" %}
18461 ins_encode %{
18462 InternalAddress addr = $constantaddress(vreplicate_imm(T_LONG, $con$$constant, VM_Version::supports_sse3() ? 1 : 2));
18463 int vlen = Matcher::vector_length_in_bytes(this);
18464 __ load_constant_vector(T_LONG, $dst$$XMMRegister, addr, vlen);
18465 %}
18466 ins_pipe( pipe_slow );
18467 %}
18468
18469 instruct ReplL_zero(vec dst, immL0 zero) %{
18470 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18471 match(Set dst (Replicate zero));
18472 format %{ "replicateL $dst,$zero" %}
18473 ins_encode %{
18474 int vlen_enc = vector_length_encoding(this);
18475 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vl()) {
18476 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18477 } else {
18478 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
18479 }
18480 %}
18481 ins_pipe( fpu_reg_reg );
18482 %}
18483
18484 instruct ReplL_M1(vec dst, immL_M1 con) %{
18485 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18486 match(Set dst (Replicate con));
18487 format %{ "vallones $dst" %}
18488 ins_encode %{
18489 int vector_len = vector_length_encoding(this);
18490 __ vallones($dst$$XMMRegister, vector_len);
18491 %}
18492 ins_pipe( pipe_slow );
18493 %}
18494
18495 // ====================ReplicateF=======================================
18496
18497 instruct vReplF_reg(vec dst, vlRegF src) %{
18498 predicate(UseAVX > 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
18499 match(Set dst (Replicate src));
18500 format %{ "replicateF $dst,$src" %}
18501 ins_encode %{
18502 uint vlen = Matcher::vector_length(this);
18503 int vlen_enc = vector_length_encoding(this);
18504 if (vlen <= 4) {
18505 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 0x00, Assembler::AVX_128bit);
18506 } else if (VM_Version::supports_avx2()) {
18507 __ vbroadcastss($dst$$XMMRegister, $src$$XMMRegister, vlen_enc); // reg-to-reg variant requires AVX2
18508 } else {
18509 assert(vlen == 8, "sanity");
18510 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 0x00, Assembler::AVX_128bit);
18511 __ vinsertf128_high($dst$$XMMRegister, $dst$$XMMRegister);
18512 }
18513 %}
18514 ins_pipe( pipe_slow );
18515 %}
18516
18517 instruct ReplF_reg(vec dst, vlRegF src) %{
18518 predicate(UseAVX == 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
18519 match(Set dst (Replicate src));
18520 format %{ "replicateF $dst,$src" %}
18521 ins_encode %{
18522 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x00);
18523 %}
18524 ins_pipe( pipe_slow );
18525 %}
18526
18527 instruct ReplF_mem(vec dst, memory mem) %{
18528 predicate(UseAVX > 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
18529 match(Set dst (Replicate (LoadF mem)));
18530 format %{ "replicateF $dst,$mem" %}
18531 ins_encode %{
18532 int vlen_enc = vector_length_encoding(this);
18533 __ vbroadcastss($dst$$XMMRegister, $mem$$Address, vlen_enc);
18534 %}
18535 ins_pipe( pipe_slow );
18536 %}
18537
18538 // Replicate float scalar immediate to be vector by loading from const table.
18539 instruct ReplF_imm(vec dst, immF con) %{
18540 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
18541 match(Set dst (Replicate con));
18542 format %{ "replicateF $dst,$con" %}
18543 ins_encode %{
18544 InternalAddress addr = $constantaddress(vreplicate_imm(T_FLOAT, $con$$constant,
18545 VM_Version::supports_sse3() ? (VM_Version::supports_avx() ? 1 : 2) : 4));
18546 int vlen = Matcher::vector_length_in_bytes(this);
18547 __ load_constant_vector(T_FLOAT, $dst$$XMMRegister, addr, vlen);
18548 %}
18549 ins_pipe( pipe_slow );
18550 %}
18551
18552 instruct ReplF_zero(vec dst, immF0 zero) %{
18553 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
18554 match(Set dst (Replicate zero));
18555 format %{ "replicateF $dst,$zero" %}
18556 ins_encode %{
18557 int vlen_enc = vector_length_encoding(this);
18558 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vldq()) {
18559 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18560 } else {
18561 __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
18562 }
18563 %}
18564 ins_pipe( fpu_reg_reg );
18565 %}
18566
18567 // ====================ReplicateD=======================================
18568
18569 // Replicate double (8 bytes) scalar to be vector
18570 instruct vReplD_reg(vec dst, vlRegD src) %{
18571 predicate(UseSSE >= 3 && Matcher::vector_element_basic_type(n) == T_DOUBLE);
18572 match(Set dst (Replicate src));
18573 format %{ "replicateD $dst,$src" %}
18574 ins_encode %{
18575 uint vlen = Matcher::vector_length(this);
18576 int vlen_enc = vector_length_encoding(this);
18577 if (vlen <= 2) {
18578 __ movddup($dst$$XMMRegister, $src$$XMMRegister);
18579 } else if (VM_Version::supports_avx2()) {
18580 __ vbroadcastsd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc); // reg-to-reg variant requires AVX2
18581 } else {
18582 assert(vlen == 4, "sanity");
18583 __ movddup($dst$$XMMRegister, $src$$XMMRegister);
18584 __ vinsertf128_high($dst$$XMMRegister, $dst$$XMMRegister);
18585 }
18586 %}
18587 ins_pipe( pipe_slow );
18588 %}
18589
18590 instruct ReplD_reg(vec dst, vlRegD src) %{
18591 predicate(UseSSE < 3 && Matcher::vector_element_basic_type(n) == T_DOUBLE);
18592 match(Set dst (Replicate src));
18593 format %{ "replicateD $dst,$src" %}
18594 ins_encode %{
18595 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x44);
18596 %}
18597 ins_pipe( pipe_slow );
18598 %}
18599
18600 instruct ReplD_mem(vec dst, memory mem) %{
18601 predicate(UseSSE >= 3 && Matcher::vector_element_basic_type(n) == T_DOUBLE);
18602 match(Set dst (Replicate (LoadD mem)));
18603 format %{ "replicateD $dst,$mem" %}
18604 ins_encode %{
18605 if (Matcher::vector_length(this) >= 4) {
18606 int vlen_enc = vector_length_encoding(this);
18607 __ vbroadcastsd($dst$$XMMRegister, $mem$$Address, vlen_enc);
18608 } else {
18609 __ movddup($dst$$XMMRegister, $mem$$Address);
18610 }
18611 %}
18612 ins_pipe( pipe_slow );
18613 %}
18614
18615 // Replicate double (8 byte) scalar immediate to be vector by loading from const table.
18616 instruct ReplD_imm(vec dst, immD con) %{
18617 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
18618 match(Set dst (Replicate con));
18619 format %{ "replicateD $dst,$con" %}
18620 ins_encode %{
18621 InternalAddress addr = $constantaddress(vreplicate_imm(T_DOUBLE, $con$$constant, VM_Version::supports_sse3() ? 1 : 2));
18622 int vlen = Matcher::vector_length_in_bytes(this);
18623 __ load_constant_vector(T_DOUBLE, $dst$$XMMRegister, addr, vlen);
18624 %}
18625 ins_pipe( pipe_slow );
18626 %}
18627
18628 instruct ReplD_zero(vec dst, immD0 zero) %{
18629 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
18630 match(Set dst (Replicate zero));
18631 format %{ "replicateD $dst,$zero" %}
18632 ins_encode %{
18633 int vlen_enc = vector_length_encoding(this);
18634 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vldq()) {
18635 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18636 } else {
18637 __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
18638 }
18639 %}
18640 ins_pipe( fpu_reg_reg );
18641 %}
18642
18643 // ====================VECTOR INSERT=======================================
18644
18645 instruct insert(vec dst, rRegI val, immU8 idx) %{
18646 predicate(Matcher::vector_length_in_bytes(n) < 32);
18647 match(Set dst (VectorInsert (Binary dst val) idx));
18648 format %{ "vector_insert $dst,$val,$idx" %}
18649 ins_encode %{
18650 assert(UseSSE >= 4, "required");
18651 assert(Matcher::vector_length_in_bytes(this) >= 8, "required");
18652
18653 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18654
18655 assert(is_integral_type(elem_bt), "");
18656 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18657
18658 __ insert(elem_bt, $dst$$XMMRegister, $val$$Register, $idx$$constant);
18659 %}
18660 ins_pipe( pipe_slow );
18661 %}
18662
18663 instruct insert32(vec dst, vec src, rRegI val, immU8 idx, vec vtmp) %{
18664 predicate(Matcher::vector_length_in_bytes(n) == 32);
18665 match(Set dst (VectorInsert (Binary src val) idx));
18666 effect(TEMP vtmp);
18667 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18668 ins_encode %{
18669 int vlen_enc = Assembler::AVX_256bit;
18670 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18671 int elem_per_lane = 16/type2aelembytes(elem_bt);
18672 int log2epr = log2(elem_per_lane);
18673
18674 assert(is_integral_type(elem_bt), "sanity");
18675 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18676
18677 uint x_idx = $idx$$constant & right_n_bits(log2epr);
18678 uint y_idx = ($idx$$constant >> log2epr) & 1;
18679 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18680 __ vinsert(elem_bt, $vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
18681 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18682 %}
18683 ins_pipe( pipe_slow );
18684 %}
18685
18686 instruct insert64(vec dst, vec src, rRegI val, immU8 idx, legVec vtmp) %{
18687 predicate(Matcher::vector_length_in_bytes(n) == 64);
18688 match(Set dst (VectorInsert (Binary src val) idx));
18689 effect(TEMP vtmp);
18690 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18691 ins_encode %{
18692 assert(UseAVX > 2, "sanity");
18693
18694 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18695 int elem_per_lane = 16/type2aelembytes(elem_bt);
18696 int log2epr = log2(elem_per_lane);
18697
18698 assert(is_integral_type(elem_bt), "");
18699 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18700
18701 uint x_idx = $idx$$constant & right_n_bits(log2epr);
18702 uint y_idx = ($idx$$constant >> log2epr) & 3;
18703 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18704 __ vinsert(elem_bt, $vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
18705 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18706 %}
18707 ins_pipe( pipe_slow );
18708 %}
18709
18710 instruct insert2L(vec dst, rRegL val, immU8 idx) %{
18711 predicate(Matcher::vector_length(n) == 2);
18712 match(Set dst (VectorInsert (Binary dst val) idx));
18713 format %{ "vector_insert $dst,$val,$idx" %}
18714 ins_encode %{
18715 assert(UseSSE >= 4, "required");
18716 assert(Matcher::vector_element_basic_type(this) == T_LONG, "");
18717 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18718
18719 __ pinsrq($dst$$XMMRegister, $val$$Register, $idx$$constant);
18720 %}
18721 ins_pipe( pipe_slow );
18722 %}
18723
18724 instruct insert4L(vec dst, vec src, rRegL val, immU8 idx, vec vtmp) %{
18725 predicate(Matcher::vector_length(n) == 4);
18726 match(Set dst (VectorInsert (Binary src val) idx));
18727 effect(TEMP vtmp);
18728 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18729 ins_encode %{
18730 assert(Matcher::vector_element_basic_type(this) == T_LONG, "");
18731 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18732
18733 uint x_idx = $idx$$constant & right_n_bits(1);
18734 uint y_idx = ($idx$$constant >> 1) & 1;
18735 int vlen_enc = Assembler::AVX_256bit;
18736 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18737 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
18738 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18739 %}
18740 ins_pipe( pipe_slow );
18741 %}
18742
18743 instruct insert8L(vec dst, vec src, rRegL val, immU8 idx, legVec vtmp) %{
18744 predicate(Matcher::vector_length(n) == 8);
18745 match(Set dst (VectorInsert (Binary src val) idx));
18746 effect(TEMP vtmp);
18747 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18748 ins_encode %{
18749 assert(Matcher::vector_element_basic_type(this) == T_LONG, "sanity");
18750 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18751
18752 uint x_idx = $idx$$constant & right_n_bits(1);
18753 uint y_idx = ($idx$$constant >> 1) & 3;
18754 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18755 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
18756 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18757 %}
18758 ins_pipe( pipe_slow );
18759 %}
18760
18761 instruct insertF(vec dst, regF val, immU8 idx) %{
18762 predicate(Matcher::vector_length(n) < 8);
18763 match(Set dst (VectorInsert (Binary dst val) idx));
18764 format %{ "vector_insert $dst,$val,$idx" %}
18765 ins_encode %{
18766 assert(UseSSE >= 4, "sanity");
18767
18768 assert(Matcher::vector_element_basic_type(this) == T_FLOAT, "sanity");
18769 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18770
18771 uint x_idx = $idx$$constant & right_n_bits(2);
18772 __ insertps($dst$$XMMRegister, $val$$XMMRegister, x_idx << 4);
18773 %}
18774 ins_pipe( pipe_slow );
18775 %}
18776
18777 instruct vinsertF(vec dst, vec src, regF val, immU8 idx, vec vtmp) %{
18778 predicate(Matcher::vector_length(n) >= 8);
18779 match(Set dst (VectorInsert (Binary src val) idx));
18780 effect(TEMP vtmp);
18781 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18782 ins_encode %{
18783 assert(Matcher::vector_element_basic_type(this) == T_FLOAT, "sanity");
18784 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18785
18786 int vlen = Matcher::vector_length(this);
18787 uint x_idx = $idx$$constant & right_n_bits(2);
18788 if (vlen == 8) {
18789 uint y_idx = ($idx$$constant >> 2) & 1;
18790 int vlen_enc = Assembler::AVX_256bit;
18791 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18792 __ vinsertps($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$XMMRegister, x_idx << 4);
18793 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18794 } else {
18795 assert(vlen == 16, "sanity");
18796 uint y_idx = ($idx$$constant >> 2) & 3;
18797 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18798 __ vinsertps($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$XMMRegister, x_idx << 4);
18799 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18800 }
18801 %}
18802 ins_pipe( pipe_slow );
18803 %}
18804
18805 instruct insert2D(vec dst, regD val, immU8 idx, rRegL tmp) %{
18806 predicate(Matcher::vector_length(n) == 2);
18807 match(Set dst (VectorInsert (Binary dst val) idx));
18808 effect(TEMP tmp);
18809 format %{ "vector_insert $dst,$val,$idx\t!using $tmp as TEMP" %}
18810 ins_encode %{
18811 assert(UseSSE >= 4, "sanity");
18812 assert(Matcher::vector_element_basic_type(this) == T_DOUBLE, "sanity");
18813 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18814
18815 __ movq($tmp$$Register, $val$$XMMRegister);
18816 __ pinsrq($dst$$XMMRegister, $tmp$$Register, $idx$$constant);
18817 %}
18818 ins_pipe( pipe_slow );
18819 %}
18820
18821 instruct insert4D(vec dst, vec src, regD val, immU8 idx, rRegL tmp, vec vtmp) %{
18822 predicate(Matcher::vector_length(n) == 4);
18823 match(Set dst (VectorInsert (Binary src val) idx));
18824 effect(TEMP vtmp, TEMP tmp);
18825 format %{ "vector_insert $dst,$src,$val,$idx\t!using $tmp, $vtmp as TEMP" %}
18826 ins_encode %{
18827 assert(Matcher::vector_element_basic_type(this) == T_DOUBLE, "sanity");
18828 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18829
18830 uint x_idx = $idx$$constant & right_n_bits(1);
18831 uint y_idx = ($idx$$constant >> 1) & 1;
18832 int vlen_enc = Assembler::AVX_256bit;
18833 __ movq($tmp$$Register, $val$$XMMRegister);
18834 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18835 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $tmp$$Register, x_idx);
18836 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18837 %}
18838 ins_pipe( pipe_slow );
18839 %}
18840
18841 instruct insert8D(vec dst, vec src, regD val, immI idx, rRegL tmp, legVec vtmp) %{
18842 predicate(Matcher::vector_length(n) == 8);
18843 match(Set dst (VectorInsert (Binary src val) idx));
18844 effect(TEMP tmp, TEMP vtmp);
18845 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18846 ins_encode %{
18847 assert(Matcher::vector_element_basic_type(this) == T_DOUBLE, "sanity");
18848 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18849
18850 uint x_idx = $idx$$constant & right_n_bits(1);
18851 uint y_idx = ($idx$$constant >> 1) & 3;
18852 __ movq($tmp$$Register, $val$$XMMRegister);
18853 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18854 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $tmp$$Register, x_idx);
18855 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18856 %}
18857 ins_pipe( pipe_slow );
18858 %}
18859
18860 // ====================REDUCTION ARITHMETIC=======================================
18861
18862 // =======================Int Reduction==========================================
18863
18864 instruct reductionI(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
18865 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_INT); // src2
18866 match(Set dst (AddReductionVI src1 src2));
18867 match(Set dst (MulReductionVI src1 src2));
18868 match(Set dst (AndReductionV src1 src2));
18869 match(Set dst ( OrReductionV src1 src2));
18870 match(Set dst (XorReductionV src1 src2));
18871 match(Set dst (MinReductionV src1 src2));
18872 match(Set dst (MaxReductionV src1 src2));
18873 match(Set dst (UMinReductionV src1 src2));
18874 match(Set dst (UMaxReductionV src1 src2));
18875 effect(TEMP vtmp1, TEMP vtmp2);
18876 format %{ "vector_reduction_int $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
18877 ins_encode %{
18878 int opcode = this->ideal_Opcode();
18879 int vlen = Matcher::vector_length(this, $src2);
18880 __ reduceI(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
18881 %}
18882 ins_pipe( pipe_slow );
18883 %}
18884
18885 // =======================Long Reduction==========================================
18886
18887 instruct reductionL(rRegL dst, rRegL src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
18888 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG && !VM_Version::supports_avx512dq());
18889 match(Set dst (AddReductionVL src1 src2));
18890 match(Set dst (MulReductionVL src1 src2));
18891 match(Set dst (AndReductionV src1 src2));
18892 match(Set dst ( OrReductionV src1 src2));
18893 match(Set dst (XorReductionV src1 src2));
18894 match(Set dst (MinReductionV src1 src2));
18895 match(Set dst (MaxReductionV src1 src2));
18896 match(Set dst (UMinReductionV src1 src2));
18897 match(Set dst (UMaxReductionV src1 src2));
18898 effect(TEMP vtmp1, TEMP vtmp2);
18899 format %{ "vector_reduction_long $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
18900 ins_encode %{
18901 int opcode = this->ideal_Opcode();
18902 int vlen = Matcher::vector_length(this, $src2);
18903 __ reduceL(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
18904 %}
18905 ins_pipe( pipe_slow );
18906 %}
18907
18908 instruct reductionL_avx512dq(rRegL dst, rRegL src1, vec src2, vec vtmp1, vec vtmp2) %{
18909 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG && VM_Version::supports_avx512dq());
18910 match(Set dst (AddReductionVL src1 src2));
18911 match(Set dst (MulReductionVL src1 src2));
18912 match(Set dst (AndReductionV src1 src2));
18913 match(Set dst ( OrReductionV src1 src2));
18914 match(Set dst (XorReductionV src1 src2));
18915 match(Set dst (MinReductionV src1 src2));
18916 match(Set dst (MaxReductionV src1 src2));
18917 match(Set dst (UMinReductionV src1 src2));
18918 match(Set dst (UMaxReductionV src1 src2));
18919 effect(TEMP vtmp1, TEMP vtmp2);
18920 format %{ "vector_reduction_long $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
18921 ins_encode %{
18922 int opcode = this->ideal_Opcode();
18923 int vlen = Matcher::vector_length(this, $src2);
18924 __ reduceL(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
18925 %}
18926 ins_pipe( pipe_slow );
18927 %}
18928
18929 // =======================Float Reduction==========================================
18930
18931 instruct reductionF128(regF dst, vec src, vec vtmp) %{
18932 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) <= 4); // src
18933 match(Set dst (AddReductionVF dst src));
18934 match(Set dst (MulReductionVF dst src));
18935 effect(TEMP dst, TEMP vtmp);
18936 format %{ "vector_reduction_float $dst,$src ; using $vtmp as TEMP" %}
18937 ins_encode %{
18938 int opcode = this->ideal_Opcode();
18939 int vlen = Matcher::vector_length(this, $src);
18940 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister);
18941 %}
18942 ins_pipe( pipe_slow );
18943 %}
18944
18945 instruct reduction8F(regF dst, vec src, vec vtmp1, vec vtmp2) %{
18946 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src
18947 match(Set dst (AddReductionVF dst src));
18948 match(Set dst (MulReductionVF dst src));
18949 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
18950 format %{ "vector_reduction_float $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
18951 ins_encode %{
18952 int opcode = this->ideal_Opcode();
18953 int vlen = Matcher::vector_length(this, $src);
18954 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
18955 %}
18956 ins_pipe( pipe_slow );
18957 %}
18958
18959 instruct reduction16F(regF dst, legVec src, legVec vtmp1, legVec vtmp2) %{
18960 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 16); // src
18961 match(Set dst (AddReductionVF dst src));
18962 match(Set dst (MulReductionVF dst src));
18963 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
18964 format %{ "vector_reduction_float $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
18965 ins_encode %{
18966 int opcode = this->ideal_Opcode();
18967 int vlen = Matcher::vector_length(this, $src);
18968 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
18969 %}
18970 ins_pipe( pipe_slow );
18971 %}
18972
18973
18974 instruct unordered_reduction2F(regF dst, regF src1, vec src2) %{
18975 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
18976 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
18977 // src1 contains reduction identity
18978 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 2); // src2
18979 match(Set dst (AddReductionVF src1 src2));
18980 match(Set dst (MulReductionVF src1 src2));
18981 effect(TEMP dst);
18982 format %{ "vector_reduction_float $dst,$src1,$src2 ;" %}
18983 ins_encode %{
18984 int opcode = this->ideal_Opcode();
18985 int vlen = Matcher::vector_length(this, $src2);
18986 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister);
18987 %}
18988 ins_pipe( pipe_slow );
18989 %}
18990
18991 instruct unordered_reduction4F(regF dst, regF src1, vec src2, vec vtmp) %{
18992 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
18993 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
18994 // src1 contains reduction identity
18995 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 4); // src2
18996 match(Set dst (AddReductionVF src1 src2));
18997 match(Set dst (MulReductionVF src1 src2));
18998 effect(TEMP dst, TEMP vtmp);
18999 format %{ "vector_reduction_float $dst,$src1,$src2 ; using $vtmp as TEMP" %}
19000 ins_encode %{
19001 int opcode = this->ideal_Opcode();
19002 int vlen = Matcher::vector_length(this, $src2);
19003 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp$$XMMRegister);
19004 %}
19005 ins_pipe( pipe_slow );
19006 %}
19007
19008 instruct unordered_reduction8F(regF dst, regF src1, vec src2, vec vtmp1, vec vtmp2) %{
19009 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
19010 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19011 // src1 contains reduction identity
19012 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src2
19013 match(Set dst (AddReductionVF src1 src2));
19014 match(Set dst (MulReductionVF src1 src2));
19015 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19016 format %{ "vector_reduction_float $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19017 ins_encode %{
19018 int opcode = this->ideal_Opcode();
19019 int vlen = Matcher::vector_length(this, $src2);
19020 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19021 %}
19022 ins_pipe( pipe_slow );
19023 %}
19024
19025 instruct unordered_reduction16F(regF dst, regF src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19026 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
19027 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19028 // src1 contains reduction identity
19029 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 16); // src2
19030 match(Set dst (AddReductionVF src1 src2));
19031 match(Set dst (MulReductionVF src1 src2));
19032 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19033 format %{ "vector_reduction_float $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19034 ins_encode %{
19035 int opcode = this->ideal_Opcode();
19036 int vlen = Matcher::vector_length(this, $src2);
19037 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19038 %}
19039 ins_pipe( pipe_slow );
19040 %}
19041
19042 // =======================Double Reduction==========================================
19043
19044 instruct reduction2D(regD dst, vec src, vec vtmp) %{
19045 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 2); // src
19046 match(Set dst (AddReductionVD dst src));
19047 match(Set dst (MulReductionVD dst src));
19048 effect(TEMP dst, TEMP vtmp);
19049 format %{ "vector_reduction_double $dst,$src ; using $vtmp as TEMP" %}
19050 ins_encode %{
19051 int opcode = this->ideal_Opcode();
19052 int vlen = Matcher::vector_length(this, $src);
19053 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister);
19054 %}
19055 ins_pipe( pipe_slow );
19056 %}
19057
19058 instruct reduction4D(regD dst, vec src, vec vtmp1, vec vtmp2) %{
19059 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 4); // src
19060 match(Set dst (AddReductionVD dst src));
19061 match(Set dst (MulReductionVD dst src));
19062 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19063 format %{ "vector_reduction_double $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
19064 ins_encode %{
19065 int opcode = this->ideal_Opcode();
19066 int vlen = Matcher::vector_length(this, $src);
19067 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19068 %}
19069 ins_pipe( pipe_slow );
19070 %}
19071
19072 instruct reduction8D(regD dst, legVec src, legVec vtmp1, legVec vtmp2) %{
19073 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src
19074 match(Set dst (AddReductionVD dst src));
19075 match(Set dst (MulReductionVD dst src));
19076 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19077 format %{ "vector_reduction_double $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
19078 ins_encode %{
19079 int opcode = this->ideal_Opcode();
19080 int vlen = Matcher::vector_length(this, $src);
19081 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19082 %}
19083 ins_pipe( pipe_slow );
19084 %}
19085
19086 instruct unordered_reduction2D(regD dst, regD src1, vec src2) %{
19087 // Non-strictly ordered floating-point add/mul reduction for doubles. This rule is
19088 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19089 // src1 contains reduction identity
19090 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 2); // src2
19091 match(Set dst (AddReductionVD src1 src2));
19092 match(Set dst (MulReductionVD src1 src2));
19093 effect(TEMP dst);
19094 format %{ "vector_reduction_double $dst,$src1,$src2 ;" %}
19095 ins_encode %{
19096 int opcode = this->ideal_Opcode();
19097 int vlen = Matcher::vector_length(this, $src2);
19098 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister);
19099 %}
19100 ins_pipe( pipe_slow );
19101 %}
19102
19103 instruct unordered_reduction4D(regD dst, regD src1, vec src2, vec vtmp) %{
19104 // Non-strictly ordered floating-point add/mul reduction for doubles. This rule is
19105 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19106 // src1 contains reduction identity
19107 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 4); // src2
19108 match(Set dst (AddReductionVD src1 src2));
19109 match(Set dst (MulReductionVD src1 src2));
19110 effect(TEMP dst, TEMP vtmp);
19111 format %{ "vector_reduction_double $dst,$src1,$src2 ; using $vtmp as TEMP" %}
19112 ins_encode %{
19113 int opcode = this->ideal_Opcode();
19114 int vlen = Matcher::vector_length(this, $src2);
19115 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp$$XMMRegister);
19116 %}
19117 ins_pipe( pipe_slow );
19118 %}
19119
19120 instruct unordered_reduction8D(regD dst, regD src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19121 // Non-strictly ordered floating-point add/mul reduction for doubles. This rule is
19122 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19123 // src1 contains reduction identity
19124 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src2
19125 match(Set dst (AddReductionVD src1 src2));
19126 match(Set dst (MulReductionVD src1 src2));
19127 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19128 format %{ "vector_reduction_double $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19129 ins_encode %{
19130 int opcode = this->ideal_Opcode();
19131 int vlen = Matcher::vector_length(this, $src2);
19132 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19133 %}
19134 ins_pipe( pipe_slow );
19135 %}
19136
19137 // =======================Byte Reduction==========================================
19138
19139 instruct reductionB(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19140 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE && !VM_Version::supports_avx512bw());
19141 match(Set dst (AddReductionVI src1 src2));
19142 match(Set dst (AndReductionV src1 src2));
19143 match(Set dst ( OrReductionV src1 src2));
19144 match(Set dst (XorReductionV src1 src2));
19145 match(Set dst (MinReductionV src1 src2));
19146 match(Set dst (MaxReductionV src1 src2));
19147 match(Set dst (UMinReductionV src1 src2));
19148 match(Set dst (UMaxReductionV src1 src2));
19149 effect(TEMP vtmp1, TEMP vtmp2);
19150 format %{ "vector_reduction_byte $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19151 ins_encode %{
19152 int opcode = this->ideal_Opcode();
19153 int vlen = Matcher::vector_length(this, $src2);
19154 __ reduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19155 %}
19156 ins_pipe( pipe_slow );
19157 %}
19158
19159 instruct reductionB_avx512bw(rRegI dst, rRegI src1, vec src2, vec vtmp1, vec vtmp2) %{
19160 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE && VM_Version::supports_avx512bw());
19161 match(Set dst (AddReductionVI src1 src2));
19162 match(Set dst (AndReductionV src1 src2));
19163 match(Set dst ( OrReductionV src1 src2));
19164 match(Set dst (XorReductionV src1 src2));
19165 match(Set dst (MinReductionV src1 src2));
19166 match(Set dst (MaxReductionV src1 src2));
19167 match(Set dst (UMinReductionV src1 src2));
19168 match(Set dst (UMaxReductionV src1 src2));
19169 effect(TEMP vtmp1, TEMP vtmp2);
19170 format %{ "vector_reduction_byte $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19171 ins_encode %{
19172 int opcode = this->ideal_Opcode();
19173 int vlen = Matcher::vector_length(this, $src2);
19174 __ reduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19175 %}
19176 ins_pipe( pipe_slow );
19177 %}
19178
19179 // =======================Short Reduction==========================================
19180
19181 instruct reductionS(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19182 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_SHORT); // src2
19183 match(Set dst (AddReductionVI src1 src2));
19184 match(Set dst (MulReductionVI src1 src2));
19185 match(Set dst (AndReductionV src1 src2));
19186 match(Set dst ( OrReductionV src1 src2));
19187 match(Set dst (XorReductionV src1 src2));
19188 match(Set dst (MinReductionV src1 src2));
19189 match(Set dst (MaxReductionV src1 src2));
19190 match(Set dst (UMinReductionV src1 src2));
19191 match(Set dst (UMaxReductionV src1 src2));
19192 effect(TEMP vtmp1, TEMP vtmp2);
19193 format %{ "vector_reduction_short $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19194 ins_encode %{
19195 int opcode = this->ideal_Opcode();
19196 int vlen = Matcher::vector_length(this, $src2);
19197 __ reduceS(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19198 %}
19199 ins_pipe( pipe_slow );
19200 %}
19201
19202 // =======================Mul Reduction==========================================
19203
19204 instruct mul_reductionB(rRegI dst, rRegI src1, vec src2, vec vtmp1, vec vtmp2) %{
19205 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE &&
19206 Matcher::vector_length(n->in(2)) <= 32); // src2
19207 match(Set dst (MulReductionVI src1 src2));
19208 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19209 format %{ "vector_mul_reduction_byte $dst,$src1,$src2; using $vtmp1, $vtmp2 as TEMP" %}
19210 ins_encode %{
19211 int opcode = this->ideal_Opcode();
19212 int vlen = Matcher::vector_length(this, $src2);
19213 __ mulreduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19214 %}
19215 ins_pipe( pipe_slow );
19216 %}
19217
19218 instruct mul_reduction64B(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19219 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE &&
19220 Matcher::vector_length(n->in(2)) == 64); // src2
19221 match(Set dst (MulReductionVI src1 src2));
19222 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19223 format %{ "vector_mul_reduction_byte $dst,$src1,$src2; using $vtmp1, $vtmp2 as TEMP" %}
19224 ins_encode %{
19225 int opcode = this->ideal_Opcode();
19226 int vlen = Matcher::vector_length(this, $src2);
19227 __ mulreduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19228 %}
19229 ins_pipe( pipe_slow );
19230 %}
19231
19232 //--------------------Min/Max Float Reduction --------------------
19233 // Float Min Reduction
19234 instruct minmax_reduction2F(legRegF dst, immF src1, legVec src2, legVec tmp, legVec atmp,
19235 legVec btmp, legVec xmm_1, rFlagsReg cr) %{
19236 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19237 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19238 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19239 Matcher::vector_length(n->in(2)) == 2);
19240 match(Set dst (MinReductionV src1 src2));
19241 match(Set dst (MaxReductionV src1 src2));
19242 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_1, KILL cr);
19243 format %{ "vector_minmax2F_reduction $dst,$src1,$src2 ; using $tmp, $atmp, $btmp, $xmm_1 as TEMP" %}
19244 ins_encode %{
19245 assert(UseAVX > 0, "sanity");
19246
19247 int opcode = this->ideal_Opcode();
19248 int vlen = Matcher::vector_length(this, $src2);
19249 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, $tmp$$XMMRegister,
19250 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_1$$XMMRegister);
19251 %}
19252 ins_pipe( pipe_slow );
19253 %}
19254
19255 instruct minmax_reductionF(legRegF dst, immF src1, legVec src2, legVec tmp, legVec atmp,
19256 legVec btmp, legVec xmm_0, legVec xmm_1, rFlagsReg cr) %{
19257 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19258 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19259 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19260 Matcher::vector_length(n->in(2)) >= 4);
19261 match(Set dst (MinReductionV src1 src2));
19262 match(Set dst (MaxReductionV src1 src2));
19263 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_0, TEMP xmm_1, KILL cr);
19264 format %{ "vector_minmaxF_reduction $dst,$src1,$src2 ; using $tmp, $atmp, $btmp, $xmm_0, $xmm_1 as TEMP" %}
19265 ins_encode %{
19266 assert(UseAVX > 0, "sanity");
19267
19268 int opcode = this->ideal_Opcode();
19269 int vlen = Matcher::vector_length(this, $src2);
19270 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, $tmp$$XMMRegister,
19271 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_0$$XMMRegister, $xmm_1$$XMMRegister);
19272 %}
19273 ins_pipe( pipe_slow );
19274 %}
19275
19276 instruct minmax_reduction2F_av(legRegF dst, legVec src, legVec tmp, legVec atmp,
19277 legVec btmp, legVec xmm_1, rFlagsReg cr) %{
19278 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19279 Matcher::vector_length(n->in(2)) == 2);
19280 match(Set dst (MinReductionV dst src));
19281 match(Set dst (MaxReductionV dst src));
19282 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_1, KILL cr);
19283 format %{ "vector_minmax2F_reduction $dst,$src ; using $tmp, $atmp, $btmp, $xmm_1 as TEMP" %}
19284 ins_encode %{
19285 assert(UseAVX > 0, "sanity");
19286
19287 int opcode = this->ideal_Opcode();
19288 int vlen = Matcher::vector_length(this, $src);
19289 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, $tmp$$XMMRegister,
19290 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_1$$XMMRegister);
19291 %}
19292 ins_pipe( pipe_slow );
19293 %}
19294
19295
19296 instruct minmax_reductionF_av(legRegF dst, legVec src, legVec tmp, legVec atmp, legVec btmp,
19297 legVec xmm_0, legVec xmm_1, rFlagsReg cr) %{
19298 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19299 Matcher::vector_length(n->in(2)) >= 4);
19300 match(Set dst (MinReductionV dst src));
19301 match(Set dst (MaxReductionV dst src));
19302 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_0, TEMP xmm_1, KILL cr);
19303 format %{ "vector_minmaxF_reduction $dst,$src ; using $tmp, $atmp, $btmp, $xmm_0, $xmm_1 as TEMP" %}
19304 ins_encode %{
19305 assert(UseAVX > 0, "sanity");
19306
19307 int opcode = this->ideal_Opcode();
19308 int vlen = Matcher::vector_length(this, $src);
19309 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, $tmp$$XMMRegister,
19310 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_0$$XMMRegister, $xmm_1$$XMMRegister);
19311 %}
19312 ins_pipe( pipe_slow );
19313 %}
19314
19315 instruct minmax_reduction2F_avx10_2(regF dst, immF src1, vec src2, vec xtmp1) %{
19316 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19317 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19318 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19319 Matcher::vector_length(n->in(2)) == 2);
19320 match(Set dst (MinReductionV src1 src2));
19321 match(Set dst (MaxReductionV src1 src2));
19322 effect(TEMP dst, TEMP xtmp1);
19323 format %{ "vector_minmax_reduction $dst, $src1, $src2 \t; using $xtmp1 as TEMP" %}
19324 ins_encode %{
19325 int opcode = this->ideal_Opcode();
19326 int vlen = Matcher::vector_length(this, $src2);
19327 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister,
19328 xnoreg, xnoreg, xnoreg, $xtmp1$$XMMRegister);
19329 %}
19330 ins_pipe( pipe_slow );
19331 %}
19332
19333 instruct minmax_reductionF_avx10_2(regF dst, immF src1, vec src2, vec xtmp1, vec xtmp2) %{
19334 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19335 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19336 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19337 Matcher::vector_length(n->in(2)) >= 4);
19338 match(Set dst (MinReductionV src1 src2));
19339 match(Set dst (MaxReductionV src1 src2));
19340 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19341 format %{ "vector_minmax_reduction $dst, $src1, $src2 \t; using $xtmp1 and $xtmp2 as TEMP" %}
19342 ins_encode %{
19343 int opcode = this->ideal_Opcode();
19344 int vlen = Matcher::vector_length(this, $src2);
19345 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, xnoreg, xnoreg,
19346 xnoreg, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19347 %}
19348 ins_pipe( pipe_slow );
19349 %}
19350
19351 instruct minmax_reduction2F_av_avx10_2(regF dst, vec src, vec xtmp1) %{
19352 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19353 Matcher::vector_length(n->in(2)) == 2);
19354 match(Set dst (MinReductionV dst src));
19355 match(Set dst (MaxReductionV dst src));
19356 effect(TEMP dst, TEMP xtmp1);
19357 format %{ "vector_minmax2F_reduction $dst, $src \t; using $xtmp1 as TEMP" %}
19358 ins_encode %{
19359 int opcode = this->ideal_Opcode();
19360 int vlen = Matcher::vector_length(this, $src);
19361 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, xnoreg, xnoreg, xnoreg,
19362 $xtmp1$$XMMRegister);
19363 %}
19364 ins_pipe( pipe_slow );
19365 %}
19366
19367 instruct minmax_reductionF_av_avx10_2(regF dst, vec src, vec xtmp1, vec xtmp2) %{
19368 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19369 Matcher::vector_length(n->in(2)) >= 4);
19370 match(Set dst (MinReductionV dst src));
19371 match(Set dst (MaxReductionV dst src));
19372 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19373 format %{ "vector_minmax2F_reduction $dst, $src \t; using $xtmp1 and $xtmp2 as TEMP" %}
19374 ins_encode %{
19375 int opcode = this->ideal_Opcode();
19376 int vlen = Matcher::vector_length(this, $src);
19377 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, xnoreg, xnoreg, xnoreg,
19378 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19379 %}
19380 ins_pipe( pipe_slow );
19381 %}
19382
19383 //--------------------Min Double Reduction --------------------
19384 instruct minmax_reduction2D(legRegD dst, immD src1, legVec src2, legVec tmp1, legVec tmp2,
19385 legVec tmp3, legVec tmp4, rFlagsReg cr) %{
19386 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19387 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19388 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19389 Matcher::vector_length(n->in(2)) == 2);
19390 match(Set dst (MinReductionV src1 src2));
19391 match(Set dst (MaxReductionV src1 src2));
19392 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
19393 format %{ "vector_minmax2D_reduction $dst,$src1,$src2 ; using $tmp1, $tmp2, $tmp3, $tmp4 as TEMP" %}
19394 ins_encode %{
19395 assert(UseAVX > 0, "sanity");
19396
19397 int opcode = this->ideal_Opcode();
19398 int vlen = Matcher::vector_length(this, $src2);
19399 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister,
19400 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister);
19401 %}
19402 ins_pipe( pipe_slow );
19403 %}
19404
19405 instruct minmax_reductionD(legRegD dst, immD src1, legVec src2, legVec tmp1, legVec tmp2,
19406 legVec tmp3, legVec tmp4, legVec tmp5, rFlagsReg cr) %{
19407 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19408 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19409 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19410 Matcher::vector_length(n->in(2)) >= 4);
19411 match(Set dst (MinReductionV src1 src2));
19412 match(Set dst (MaxReductionV src1 src2));
19413 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, KILL cr);
19414 format %{ "vector_minmaxD_reduction $dst,$src1,$src2 ; using $tmp1, $tmp2, $tmp3, $tmp4, $tmp5 as TEMP" %}
19415 ins_encode %{
19416 assert(UseAVX > 0, "sanity");
19417
19418 int opcode = this->ideal_Opcode();
19419 int vlen = Matcher::vector_length(this, $src2);
19420 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister,
19421 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister, $tmp5$$XMMRegister);
19422 %}
19423 ins_pipe( pipe_slow );
19424 %}
19425
19426
19427 instruct minmax_reduction2D_av(legRegD dst, legVec src, legVec tmp1, legVec tmp2,
19428 legVec tmp3, legVec tmp4, rFlagsReg cr) %{
19429 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19430 Matcher::vector_length(n->in(2)) == 2);
19431 match(Set dst (MinReductionV dst src));
19432 match(Set dst (MaxReductionV dst src));
19433 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
19434 format %{ "vector_minmax2D_reduction $dst,$src ; using $tmp1, $tmp2, $tmp3, $tmp4 as TEMP" %}
19435 ins_encode %{
19436 assert(UseAVX > 0, "sanity");
19437
19438 int opcode = this->ideal_Opcode();
19439 int vlen = Matcher::vector_length(this, $src);
19440 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19441 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister);
19442 %}
19443 ins_pipe( pipe_slow );
19444 %}
19445
19446 instruct minmax_reductionD_av(legRegD dst, legVec src, legVec tmp1, legVec tmp2, legVec tmp3,
19447 legVec tmp4, legVec tmp5, rFlagsReg cr) %{
19448 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19449 Matcher::vector_length(n->in(2)) >= 4);
19450 match(Set dst (MinReductionV dst src));
19451 match(Set dst (MaxReductionV dst src));
19452 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, KILL cr);
19453 format %{ "vector_minmaxD_reduction $dst,$src ; using $tmp1, $tmp2, $tmp3, $tmp4, $tmp5 as TEMP" %}
19454 ins_encode %{
19455 assert(UseAVX > 0, "sanity");
19456
19457 int opcode = this->ideal_Opcode();
19458 int vlen = Matcher::vector_length(this, $src);
19459 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19460 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister, $tmp5$$XMMRegister);
19461 %}
19462 ins_pipe( pipe_slow );
19463 %}
19464
19465 instruct minmax_reduction2D_avx10_2(regD dst, immD src1, vec src2, vec xtmp1) %{
19466 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19467 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19468 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19469 Matcher::vector_length(n->in(2)) == 2);
19470 match(Set dst (MinReductionV src1 src2));
19471 match(Set dst (MaxReductionV src1 src2));
19472 effect(TEMP dst, TEMP xtmp1);
19473 format %{ "vector_minmax2D_reduction $dst, $src1, $src2 ; using $xtmp1 as TEMP" %}
19474 ins_encode %{
19475 int opcode = this->ideal_Opcode();
19476 int vlen = Matcher::vector_length(this, $src2);
19477 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, xnoreg,
19478 xnoreg, xnoreg, $xtmp1$$XMMRegister);
19479 %}
19480 ins_pipe( pipe_slow );
19481 %}
19482
19483 instruct minmax_reductionD_avx10_2(regD dst, immD src1, vec src2, vec xtmp1, vec xtmp2) %{
19484 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19485 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19486 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19487 Matcher::vector_length(n->in(2)) >= 4);
19488 match(Set dst (MinReductionV src1 src2));
19489 match(Set dst (MaxReductionV src1 src2));
19490 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19491 format %{ "vector_minmaxD_reduction $dst, $src1, $src2 ; using $xtmp1 and $xtmp2 as TEMP" %}
19492 ins_encode %{
19493 int opcode = this->ideal_Opcode();
19494 int vlen = Matcher::vector_length(this, $src2);
19495 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, xnoreg, xnoreg,
19496 xnoreg, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19497 %}
19498 ins_pipe( pipe_slow );
19499 %}
19500
19501
19502 instruct minmax_reduction2D_av_avx10_2(regD dst, vec src, vec xtmp1) %{
19503 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19504 Matcher::vector_length(n->in(2)) == 2);
19505 match(Set dst (MinReductionV dst src));
19506 match(Set dst (MaxReductionV dst src));
19507 effect(TEMP dst, TEMP xtmp1);
19508 format %{ "vector_minmax2D_reduction $dst, $src ; using $xtmp1 as TEMP" %}
19509 ins_encode %{
19510 int opcode = this->ideal_Opcode();
19511 int vlen = Matcher::vector_length(this, $src);
19512 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19513 xnoreg, xnoreg, xnoreg, $xtmp1$$XMMRegister);
19514 %}
19515 ins_pipe( pipe_slow );
19516 %}
19517
19518 instruct minmax_reductionD_av_avx10_2(regD dst, vec src, vec xtmp1, vec xtmp2) %{
19519 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19520 Matcher::vector_length(n->in(2)) >= 4);
19521 match(Set dst (MinReductionV dst src));
19522 match(Set dst (MaxReductionV dst src));
19523 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19524 format %{ "vector_minmaxD_reduction $dst, $src ; using $xtmp1 and $xtmp2 as TEMP" %}
19525 ins_encode %{
19526 int opcode = this->ideal_Opcode();
19527 int vlen = Matcher::vector_length(this, $src);
19528 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19529 xnoreg, xnoreg, xnoreg, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19530 %}
19531 ins_pipe( pipe_slow );
19532 %}
19533
19534 // ====================VECTOR ARITHMETIC=======================================
19535
19536 // --------------------------------- ADD --------------------------------------
19537
19538 // Bytes vector add
19539 instruct vaddB(vec dst, vec src) %{
19540 predicate(UseAVX == 0);
19541 match(Set dst (AddVB dst src));
19542 format %{ "paddb $dst,$src\t! add packedB" %}
19543 ins_encode %{
19544 __ paddb($dst$$XMMRegister, $src$$XMMRegister);
19545 %}
19546 ins_pipe( pipe_slow );
19547 %}
19548
19549 instruct vaddB_reg(vec dst, vec src1, vec src2) %{
19550 predicate(UseAVX > 0);
19551 match(Set dst (AddVB src1 src2));
19552 format %{ "vpaddb $dst,$src1,$src2\t! add packedB" %}
19553 ins_encode %{
19554 int vlen_enc = vector_length_encoding(this);
19555 __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19556 %}
19557 ins_pipe( pipe_slow );
19558 %}
19559
19560 instruct vaddB_mem(vec dst, vec src, memory mem) %{
19561 predicate((UseAVX > 0) &&
19562 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19563 match(Set dst (AddVB src (LoadVector mem)));
19564 format %{ "vpaddb $dst,$src,$mem\t! add packedB" %}
19565 ins_encode %{
19566 int vlen_enc = vector_length_encoding(this);
19567 __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19568 %}
19569 ins_pipe( pipe_slow );
19570 %}
19571
19572 // Shorts/Chars vector add
19573 instruct vaddS(vec dst, vec src) %{
19574 predicate(UseAVX == 0);
19575 match(Set dst (AddVS dst src));
19576 format %{ "paddw $dst,$src\t! add packedS" %}
19577 ins_encode %{
19578 __ paddw($dst$$XMMRegister, $src$$XMMRegister);
19579 %}
19580 ins_pipe( pipe_slow );
19581 %}
19582
19583 instruct vaddS_reg(vec dst, vec src1, vec src2) %{
19584 predicate(UseAVX > 0);
19585 match(Set dst (AddVS src1 src2));
19586 format %{ "vpaddw $dst,$src1,$src2\t! add packedS" %}
19587 ins_encode %{
19588 int vlen_enc = vector_length_encoding(this);
19589 __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19590 %}
19591 ins_pipe( pipe_slow );
19592 %}
19593
19594 instruct vaddS_mem(vec dst, vec src, memory mem) %{
19595 predicate((UseAVX > 0) &&
19596 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19597 match(Set dst (AddVS src (LoadVector mem)));
19598 format %{ "vpaddw $dst,$src,$mem\t! add packedS" %}
19599 ins_encode %{
19600 int vlen_enc = vector_length_encoding(this);
19601 __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19602 %}
19603 ins_pipe( pipe_slow );
19604 %}
19605
19606 // Integers vector add
19607 instruct vaddI(vec dst, vec src) %{
19608 predicate(UseAVX == 0);
19609 match(Set dst (AddVI dst src));
19610 format %{ "paddd $dst,$src\t! add packedI" %}
19611 ins_encode %{
19612 __ paddd($dst$$XMMRegister, $src$$XMMRegister);
19613 %}
19614 ins_pipe( pipe_slow );
19615 %}
19616
19617 instruct vaddI_reg(vec dst, vec src1, vec src2) %{
19618 predicate(UseAVX > 0);
19619 match(Set dst (AddVI src1 src2));
19620 format %{ "vpaddd $dst,$src1,$src2\t! add packedI" %}
19621 ins_encode %{
19622 int vlen_enc = vector_length_encoding(this);
19623 __ vpaddd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19624 %}
19625 ins_pipe( pipe_slow );
19626 %}
19627
19628
19629 instruct vaddI_mem(vec dst, vec src, memory mem) %{
19630 predicate((UseAVX > 0) &&
19631 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19632 match(Set dst (AddVI src (LoadVector mem)));
19633 format %{ "vpaddd $dst,$src,$mem\t! add packedI" %}
19634 ins_encode %{
19635 int vlen_enc = vector_length_encoding(this);
19636 __ vpaddd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19637 %}
19638 ins_pipe( pipe_slow );
19639 %}
19640
19641 // Longs vector add
19642 instruct vaddL(vec dst, vec src) %{
19643 predicate(UseAVX == 0);
19644 match(Set dst (AddVL dst src));
19645 format %{ "paddq $dst,$src\t! add packedL" %}
19646 ins_encode %{
19647 __ paddq($dst$$XMMRegister, $src$$XMMRegister);
19648 %}
19649 ins_pipe( pipe_slow );
19650 %}
19651
19652 instruct vaddL_reg(vec dst, vec src1, vec src2) %{
19653 predicate(UseAVX > 0);
19654 match(Set dst (AddVL src1 src2));
19655 format %{ "vpaddq $dst,$src1,$src2\t! add packedL" %}
19656 ins_encode %{
19657 int vlen_enc = vector_length_encoding(this);
19658 __ vpaddq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19659 %}
19660 ins_pipe( pipe_slow );
19661 %}
19662
19663 instruct vaddL_mem(vec dst, vec src, memory mem) %{
19664 predicate((UseAVX > 0) &&
19665 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19666 match(Set dst (AddVL src (LoadVector mem)));
19667 format %{ "vpaddq $dst,$src,$mem\t! add packedL" %}
19668 ins_encode %{
19669 int vlen_enc = vector_length_encoding(this);
19670 __ vpaddq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19671 %}
19672 ins_pipe( pipe_slow );
19673 %}
19674
19675 // Floats vector add
19676 instruct vaddF(vec dst, vec src) %{
19677 predicate(UseAVX == 0);
19678 match(Set dst (AddVF dst src));
19679 format %{ "addps $dst,$src\t! add packedF" %}
19680 ins_encode %{
19681 __ addps($dst$$XMMRegister, $src$$XMMRegister);
19682 %}
19683 ins_pipe( pipe_slow );
19684 %}
19685
19686 instruct vaddF_reg(vec dst, vec src1, vec src2) %{
19687 predicate(UseAVX > 0);
19688 match(Set dst (AddVF src1 src2));
19689 format %{ "vaddps $dst,$src1,$src2\t! add packedF" %}
19690 ins_encode %{
19691 int vlen_enc = vector_length_encoding(this);
19692 __ vaddps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19693 %}
19694 ins_pipe( pipe_slow );
19695 %}
19696
19697 instruct vaddF_mem(vec dst, vec src, memory mem) %{
19698 predicate((UseAVX > 0) &&
19699 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19700 match(Set dst (AddVF src (LoadVector mem)));
19701 format %{ "vaddps $dst,$src,$mem\t! add packedF" %}
19702 ins_encode %{
19703 int vlen_enc = vector_length_encoding(this);
19704 __ vaddps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19705 %}
19706 ins_pipe( pipe_slow );
19707 %}
19708
19709 // Doubles vector add
19710 instruct vaddD(vec dst, vec src) %{
19711 predicate(UseAVX == 0);
19712 match(Set dst (AddVD dst src));
19713 format %{ "addpd $dst,$src\t! add packedD" %}
19714 ins_encode %{
19715 __ addpd($dst$$XMMRegister, $src$$XMMRegister);
19716 %}
19717 ins_pipe( pipe_slow );
19718 %}
19719
19720 instruct vaddD_reg(vec dst, vec src1, vec src2) %{
19721 predicate(UseAVX > 0);
19722 match(Set dst (AddVD src1 src2));
19723 format %{ "vaddpd $dst,$src1,$src2\t! add packedD" %}
19724 ins_encode %{
19725 int vlen_enc = vector_length_encoding(this);
19726 __ vaddpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19727 %}
19728 ins_pipe( pipe_slow );
19729 %}
19730
19731 instruct vaddD_mem(vec dst, vec src, memory mem) %{
19732 predicate((UseAVX > 0) &&
19733 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19734 match(Set dst (AddVD src (LoadVector mem)));
19735 format %{ "vaddpd $dst,$src,$mem\t! add packedD" %}
19736 ins_encode %{
19737 int vlen_enc = vector_length_encoding(this);
19738 __ vaddpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19739 %}
19740 ins_pipe( pipe_slow );
19741 %}
19742
19743 // --------------------------------- SUB --------------------------------------
19744
19745 // Bytes vector sub
19746 instruct vsubB(vec dst, vec src) %{
19747 predicate(UseAVX == 0);
19748 match(Set dst (SubVB dst src));
19749 format %{ "psubb $dst,$src\t! sub packedB" %}
19750 ins_encode %{
19751 __ psubb($dst$$XMMRegister, $src$$XMMRegister);
19752 %}
19753 ins_pipe( pipe_slow );
19754 %}
19755
19756 instruct vsubB_reg(vec dst, vec src1, vec src2) %{
19757 predicate(UseAVX > 0);
19758 match(Set dst (SubVB src1 src2));
19759 format %{ "vpsubb $dst,$src1,$src2\t! sub packedB" %}
19760 ins_encode %{
19761 int vlen_enc = vector_length_encoding(this);
19762 __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19763 %}
19764 ins_pipe( pipe_slow );
19765 %}
19766
19767 instruct vsubB_mem(vec dst, vec src, memory mem) %{
19768 predicate((UseAVX > 0) &&
19769 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19770 match(Set dst (SubVB src (LoadVector mem)));
19771 format %{ "vpsubb $dst,$src,$mem\t! sub packedB" %}
19772 ins_encode %{
19773 int vlen_enc = vector_length_encoding(this);
19774 __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19775 %}
19776 ins_pipe( pipe_slow );
19777 %}
19778
19779 // Shorts/Chars vector sub
19780 instruct vsubS(vec dst, vec src) %{
19781 predicate(UseAVX == 0);
19782 match(Set dst (SubVS dst src));
19783 format %{ "psubw $dst,$src\t! sub packedS" %}
19784 ins_encode %{
19785 __ psubw($dst$$XMMRegister, $src$$XMMRegister);
19786 %}
19787 ins_pipe( pipe_slow );
19788 %}
19789
19790
19791 instruct vsubS_reg(vec dst, vec src1, vec src2) %{
19792 predicate(UseAVX > 0);
19793 match(Set dst (SubVS src1 src2));
19794 format %{ "vpsubw $dst,$src1,$src2\t! sub packedS" %}
19795 ins_encode %{
19796 int vlen_enc = vector_length_encoding(this);
19797 __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19798 %}
19799 ins_pipe( pipe_slow );
19800 %}
19801
19802 instruct vsubS_mem(vec dst, vec src, memory mem) %{
19803 predicate((UseAVX > 0) &&
19804 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19805 match(Set dst (SubVS src (LoadVector mem)));
19806 format %{ "vpsubw $dst,$src,$mem\t! sub packedS" %}
19807 ins_encode %{
19808 int vlen_enc = vector_length_encoding(this);
19809 __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19810 %}
19811 ins_pipe( pipe_slow );
19812 %}
19813
19814 // Integers vector sub
19815 instruct vsubI(vec dst, vec src) %{
19816 predicate(UseAVX == 0);
19817 match(Set dst (SubVI dst src));
19818 format %{ "psubd $dst,$src\t! sub packedI" %}
19819 ins_encode %{
19820 __ psubd($dst$$XMMRegister, $src$$XMMRegister);
19821 %}
19822 ins_pipe( pipe_slow );
19823 %}
19824
19825 instruct vsubI_reg(vec dst, vec src1, vec src2) %{
19826 predicate(UseAVX > 0);
19827 match(Set dst (SubVI src1 src2));
19828 format %{ "vpsubd $dst,$src1,$src2\t! sub packedI" %}
19829 ins_encode %{
19830 int vlen_enc = vector_length_encoding(this);
19831 __ vpsubd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19832 %}
19833 ins_pipe( pipe_slow );
19834 %}
19835
19836 instruct vsubI_mem(vec dst, vec src, memory mem) %{
19837 predicate((UseAVX > 0) &&
19838 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19839 match(Set dst (SubVI src (LoadVector mem)));
19840 format %{ "vpsubd $dst,$src,$mem\t! sub packedI" %}
19841 ins_encode %{
19842 int vlen_enc = vector_length_encoding(this);
19843 __ vpsubd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19844 %}
19845 ins_pipe( pipe_slow );
19846 %}
19847
19848 // Longs vector sub
19849 instruct vsubL(vec dst, vec src) %{
19850 predicate(UseAVX == 0);
19851 match(Set dst (SubVL dst src));
19852 format %{ "psubq $dst,$src\t! sub packedL" %}
19853 ins_encode %{
19854 __ psubq($dst$$XMMRegister, $src$$XMMRegister);
19855 %}
19856 ins_pipe( pipe_slow );
19857 %}
19858
19859 instruct vsubL_reg(vec dst, vec src1, vec src2) %{
19860 predicate(UseAVX > 0);
19861 match(Set dst (SubVL src1 src2));
19862 format %{ "vpsubq $dst,$src1,$src2\t! sub packedL" %}
19863 ins_encode %{
19864 int vlen_enc = vector_length_encoding(this);
19865 __ vpsubq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19866 %}
19867 ins_pipe( pipe_slow );
19868 %}
19869
19870
19871 instruct vsubL_mem(vec dst, vec src, memory mem) %{
19872 predicate((UseAVX > 0) &&
19873 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19874 match(Set dst (SubVL src (LoadVector mem)));
19875 format %{ "vpsubq $dst,$src,$mem\t! sub packedL" %}
19876 ins_encode %{
19877 int vlen_enc = vector_length_encoding(this);
19878 __ vpsubq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19879 %}
19880 ins_pipe( pipe_slow );
19881 %}
19882
19883 // Floats vector sub
19884 instruct vsubF(vec dst, vec src) %{
19885 predicate(UseAVX == 0);
19886 match(Set dst (SubVF dst src));
19887 format %{ "subps $dst,$src\t! sub packedF" %}
19888 ins_encode %{
19889 __ subps($dst$$XMMRegister, $src$$XMMRegister);
19890 %}
19891 ins_pipe( pipe_slow );
19892 %}
19893
19894 instruct vsubF_reg(vec dst, vec src1, vec src2) %{
19895 predicate(UseAVX > 0);
19896 match(Set dst (SubVF src1 src2));
19897 format %{ "vsubps $dst,$src1,$src2\t! sub packedF" %}
19898 ins_encode %{
19899 int vlen_enc = vector_length_encoding(this);
19900 __ vsubps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19901 %}
19902 ins_pipe( pipe_slow );
19903 %}
19904
19905 instruct vsubF_mem(vec dst, vec src, memory mem) %{
19906 predicate((UseAVX > 0) &&
19907 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19908 match(Set dst (SubVF src (LoadVector mem)));
19909 format %{ "vsubps $dst,$src,$mem\t! sub packedF" %}
19910 ins_encode %{
19911 int vlen_enc = vector_length_encoding(this);
19912 __ vsubps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19913 %}
19914 ins_pipe( pipe_slow );
19915 %}
19916
19917 // Doubles vector sub
19918 instruct vsubD(vec dst, vec src) %{
19919 predicate(UseAVX == 0);
19920 match(Set dst (SubVD dst src));
19921 format %{ "subpd $dst,$src\t! sub packedD" %}
19922 ins_encode %{
19923 __ subpd($dst$$XMMRegister, $src$$XMMRegister);
19924 %}
19925 ins_pipe( pipe_slow );
19926 %}
19927
19928 instruct vsubD_reg(vec dst, vec src1, vec src2) %{
19929 predicate(UseAVX > 0);
19930 match(Set dst (SubVD src1 src2));
19931 format %{ "vsubpd $dst,$src1,$src2\t! sub packedD" %}
19932 ins_encode %{
19933 int vlen_enc = vector_length_encoding(this);
19934 __ vsubpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19935 %}
19936 ins_pipe( pipe_slow );
19937 %}
19938
19939 instruct vsubD_mem(vec dst, vec src, memory mem) %{
19940 predicate((UseAVX > 0) &&
19941 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19942 match(Set dst (SubVD src (LoadVector mem)));
19943 format %{ "vsubpd $dst,$src,$mem\t! sub packedD" %}
19944 ins_encode %{
19945 int vlen_enc = vector_length_encoding(this);
19946 __ vsubpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19947 %}
19948 ins_pipe( pipe_slow );
19949 %}
19950
19951 // --------------------------------- MUL --------------------------------------
19952
19953 // Byte vector mul
19954 instruct vmul8B(vec dst, vec src1, vec src2, vec xtmp) %{
19955 predicate(Matcher::vector_length_in_bytes(n) <= 8);
19956 match(Set dst (MulVB src1 src2));
19957 effect(TEMP dst, TEMP xtmp);
19958 format %{ "mulVB $dst, $src1, $src2\t! using $xtmp as TEMP" %}
19959 ins_encode %{
19960 assert(UseSSE > 3, "required");
19961 __ pmovsxbw($dst$$XMMRegister, $src1$$XMMRegister);
19962 __ pmovsxbw($xtmp$$XMMRegister, $src2$$XMMRegister);
19963 __ pmullw($dst$$XMMRegister, $xtmp$$XMMRegister);
19964 __ psllw($dst$$XMMRegister, 8);
19965 __ psrlw($dst$$XMMRegister, 8);
19966 __ packuswb($dst$$XMMRegister, $dst$$XMMRegister);
19967 %}
19968 ins_pipe( pipe_slow );
19969 %}
19970
19971 instruct vmulB(vec dst, vec src1, vec src2, vec xtmp) %{
19972 predicate(UseAVX == 0 && Matcher::vector_length_in_bytes(n) > 8);
19973 match(Set dst (MulVB src1 src2));
19974 effect(TEMP dst, TEMP xtmp);
19975 format %{ "mulVB $dst, $src1, $src2\t! using $xtmp as TEMP" %}
19976 ins_encode %{
19977 assert(UseSSE > 3, "required");
19978 // Odd-index elements
19979 __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
19980 __ psrlw($dst$$XMMRegister, 8);
19981 __ movdqu($xtmp$$XMMRegister, $src2$$XMMRegister);
19982 __ psrlw($xtmp$$XMMRegister, 8);
19983 __ pmullw($dst$$XMMRegister, $xtmp$$XMMRegister);
19984 __ psllw($dst$$XMMRegister, 8);
19985 // Even-index elements
19986 __ movdqu($xtmp$$XMMRegister, $src1$$XMMRegister);
19987 __ pmullw($xtmp$$XMMRegister, $src2$$XMMRegister);
19988 __ psllw($xtmp$$XMMRegister, 8);
19989 __ psrlw($xtmp$$XMMRegister, 8);
19990 // Combine
19991 __ por($dst$$XMMRegister, $xtmp$$XMMRegister);
19992 %}
19993 ins_pipe( pipe_slow );
19994 %}
19995
19996 instruct vmulB_reg(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2) %{
19997 predicate(UseAVX > 0 && Matcher::vector_length_in_bytes(n) > 8);
19998 match(Set dst (MulVB src1 src2));
19999 effect(TEMP xtmp1, TEMP xtmp2);
20000 format %{ "vmulVB $dst, $src1, $src2\t! using $xtmp1, $xtmp2 as TEMP" %}
20001 ins_encode %{
20002 int vlen_enc = vector_length_encoding(this);
20003 // Odd-index elements
20004 __ vpsrlw($xtmp2$$XMMRegister, $src1$$XMMRegister, 8, vlen_enc);
20005 __ vpsrlw($xtmp1$$XMMRegister, $src2$$XMMRegister, 8, vlen_enc);
20006 __ vpmullw($xtmp2$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20007 __ vpsllw($xtmp2$$XMMRegister, $xtmp2$$XMMRegister, 8, vlen_enc);
20008 // Even-index elements
20009 __ vpmullw($xtmp1$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20010 __ vpsllw($xtmp1$$XMMRegister, $xtmp1$$XMMRegister, 8, vlen_enc);
20011 __ vpsrlw($xtmp1$$XMMRegister, $xtmp1$$XMMRegister, 8, vlen_enc);
20012 // Combine
20013 __ vpor($dst$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20014 %}
20015 ins_pipe( pipe_slow );
20016 %}
20017
20018 // Shorts/Chars vector mul
20019 instruct vmulS(vec dst, vec src) %{
20020 predicate(UseAVX == 0);
20021 match(Set dst (MulVS dst src));
20022 format %{ "pmullw $dst,$src\t! mul packedS" %}
20023 ins_encode %{
20024 __ pmullw($dst$$XMMRegister, $src$$XMMRegister);
20025 %}
20026 ins_pipe( pipe_slow );
20027 %}
20028
20029 instruct vmulS_reg(vec dst, vec src1, vec src2) %{
20030 predicate(UseAVX > 0);
20031 match(Set dst (MulVS src1 src2));
20032 format %{ "vpmullw $dst,$src1,$src2\t! mul packedS" %}
20033 ins_encode %{
20034 int vlen_enc = vector_length_encoding(this);
20035 __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20036 %}
20037 ins_pipe( pipe_slow );
20038 %}
20039
20040 instruct vmulS_mem(vec dst, vec src, memory mem) %{
20041 predicate((UseAVX > 0) &&
20042 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20043 match(Set dst (MulVS src (LoadVector mem)));
20044 format %{ "vpmullw $dst,$src,$mem\t! mul packedS" %}
20045 ins_encode %{
20046 int vlen_enc = vector_length_encoding(this);
20047 __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20048 %}
20049 ins_pipe( pipe_slow );
20050 %}
20051
20052 // Integers vector mul
20053 instruct vmulI(vec dst, vec src) %{
20054 predicate(UseAVX == 0);
20055 match(Set dst (MulVI dst src));
20056 format %{ "pmulld $dst,$src\t! mul packedI" %}
20057 ins_encode %{
20058 assert(UseSSE > 3, "required");
20059 __ pmulld($dst$$XMMRegister, $src$$XMMRegister);
20060 %}
20061 ins_pipe( pipe_slow );
20062 %}
20063
20064 instruct vmulI_reg(vec dst, vec src1, vec src2) %{
20065 predicate(UseAVX > 0);
20066 match(Set dst (MulVI src1 src2));
20067 format %{ "vpmulld $dst,$src1,$src2\t! mul packedI" %}
20068 ins_encode %{
20069 int vlen_enc = vector_length_encoding(this);
20070 __ vpmulld($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20071 %}
20072 ins_pipe( pipe_slow );
20073 %}
20074
20075 instruct vmulI_mem(vec dst, vec src, memory mem) %{
20076 predicate((UseAVX > 0) &&
20077 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20078 match(Set dst (MulVI src (LoadVector mem)));
20079 format %{ "vpmulld $dst,$src,$mem\t! mul packedI" %}
20080 ins_encode %{
20081 int vlen_enc = vector_length_encoding(this);
20082 __ vpmulld($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20083 %}
20084 ins_pipe( pipe_slow );
20085 %}
20086
20087 // Longs vector mul
20088 instruct evmulL_reg(vec dst, vec src1, vec src2) %{
20089 predicate((Matcher::vector_length_in_bytes(n) == 64 &&
20090 VM_Version::supports_avx512dq()) ||
20091 VM_Version::supports_avx512vldq());
20092 match(Set dst (MulVL src1 src2));
20093 ins_cost(500);
20094 format %{ "evpmullq $dst,$src1,$src2\t! mul packedL" %}
20095 ins_encode %{
20096 assert(UseAVX > 2, "required");
20097 int vlen_enc = vector_length_encoding(this);
20098 __ evpmullq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20099 %}
20100 ins_pipe( pipe_slow );
20101 %}
20102
20103 instruct evmulL_mem(vec dst, vec src, memory mem) %{
20104 predicate((Matcher::vector_length_in_bytes(n) == 64 &&
20105 VM_Version::supports_avx512dq()) ||
20106 (Matcher::vector_length_in_bytes(n) > 8 &&
20107 VM_Version::supports_avx512vldq()));
20108 match(Set dst (MulVL src (LoadVector mem)));
20109 format %{ "evpmullq $dst,$src,$mem\t! mul packedL" %}
20110 ins_cost(500);
20111 ins_encode %{
20112 assert(UseAVX > 2, "required");
20113 int vlen_enc = vector_length_encoding(this);
20114 __ evpmullq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20115 %}
20116 ins_pipe( pipe_slow );
20117 %}
20118
20119 instruct vmulL(vec dst, vec src1, vec src2, vec xtmp) %{
20120 predicate(UseAVX == 0);
20121 match(Set dst (MulVL src1 src2));
20122 ins_cost(500);
20123 effect(TEMP dst, TEMP xtmp);
20124 format %{ "mulVL $dst, $src1, $src2\t! using $xtmp as TEMP" %}
20125 ins_encode %{
20126 assert(VM_Version::supports_sse4_1(), "required");
20127 // Get the lo-hi products, only the lower 32 bits is in concerns
20128 __ pshufd($xtmp$$XMMRegister, $src2$$XMMRegister, 0xB1);
20129 __ pmulld($xtmp$$XMMRegister, $src1$$XMMRegister);
20130 __ pshufd($dst$$XMMRegister, $xtmp$$XMMRegister, 0xB1);
20131 __ paddd($dst$$XMMRegister, $xtmp$$XMMRegister);
20132 __ psllq($dst$$XMMRegister, 32);
20133 // Get the lo-lo products
20134 __ movdqu($xtmp$$XMMRegister, $src1$$XMMRegister);
20135 __ pmuludq($xtmp$$XMMRegister, $src2$$XMMRegister);
20136 __ paddq($dst$$XMMRegister, $xtmp$$XMMRegister);
20137 %}
20138 ins_pipe( pipe_slow );
20139 %}
20140
20141 instruct vmulL_reg(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2) %{
20142 predicate(UseAVX > 0 &&
20143 ((Matcher::vector_length_in_bytes(n) == 64 &&
20144 !VM_Version::supports_avx512dq()) ||
20145 (Matcher::vector_length_in_bytes(n) < 64 &&
20146 !VM_Version::supports_avx512vldq())));
20147 match(Set dst (MulVL src1 src2));
20148 effect(TEMP xtmp1, TEMP xtmp2);
20149 ins_cost(500);
20150 format %{ "vmulVL $dst, $src1, $src2\t! using $xtmp1, $xtmp2 as TEMP" %}
20151 ins_encode %{
20152 int vlen_enc = vector_length_encoding(this);
20153 // Get the lo-hi products, only the lower 32 bits is in concerns
20154 __ vpshufd($xtmp1$$XMMRegister, $src2$$XMMRegister, 0xB1, vlen_enc);
20155 __ vpmulld($xtmp1$$XMMRegister, $src1$$XMMRegister, $xtmp1$$XMMRegister, vlen_enc);
20156 __ vpshufd($xtmp2$$XMMRegister, $xtmp1$$XMMRegister, 0xB1, vlen_enc);
20157 __ vpaddd($xtmp2$$XMMRegister, $xtmp2$$XMMRegister, $xtmp1$$XMMRegister, vlen_enc);
20158 __ vpsllq($xtmp2$$XMMRegister, $xtmp2$$XMMRegister, 32, vlen_enc);
20159 // Get the lo-lo products
20160 __ vpmuludq($xtmp1$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20161 __ vpaddq($dst$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20162 %}
20163 ins_pipe( pipe_slow );
20164 %}
20165
20166 instruct vmuludq_reg(vec dst, vec src1, vec src2) %{
20167 predicate(UseAVX > 0 && n->as_MulVL()->has_uint_inputs());
20168 match(Set dst (MulVL src1 src2));
20169 ins_cost(100);
20170 format %{ "vpmuludq $dst,$src1,$src2\t! muludq packedL" %}
20171 ins_encode %{
20172 int vlen_enc = vector_length_encoding(this);
20173 __ vpmuludq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20174 %}
20175 ins_pipe( pipe_slow );
20176 %}
20177
20178 instruct vmuldq_reg(vec dst, vec src1, vec src2) %{
20179 predicate(UseAVX > 0 && n->as_MulVL()->has_int_inputs());
20180 match(Set dst (MulVL src1 src2));
20181 ins_cost(100);
20182 format %{ "vpmuldq $dst,$src1,$src2\t! muldq packedL" %}
20183 ins_encode %{
20184 int vlen_enc = vector_length_encoding(this);
20185 __ vpmuldq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20186 %}
20187 ins_pipe( pipe_slow );
20188 %}
20189
20190 // Floats vector mul
20191 instruct vmulF(vec dst, vec src) %{
20192 predicate(UseAVX == 0);
20193 match(Set dst (MulVF dst src));
20194 format %{ "mulps $dst,$src\t! mul packedF" %}
20195 ins_encode %{
20196 __ mulps($dst$$XMMRegister, $src$$XMMRegister);
20197 %}
20198 ins_pipe( pipe_slow );
20199 %}
20200
20201 instruct vmulF_reg(vec dst, vec src1, vec src2) %{
20202 predicate(UseAVX > 0);
20203 match(Set dst (MulVF src1 src2));
20204 format %{ "vmulps $dst,$src1,$src2\t! mul packedF" %}
20205 ins_encode %{
20206 int vlen_enc = vector_length_encoding(this);
20207 __ vmulps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20208 %}
20209 ins_pipe( pipe_slow );
20210 %}
20211
20212 instruct vmulF_mem(vec dst, vec src, memory mem) %{
20213 predicate((UseAVX > 0) &&
20214 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20215 match(Set dst (MulVF src (LoadVector mem)));
20216 format %{ "vmulps $dst,$src,$mem\t! mul packedF" %}
20217 ins_encode %{
20218 int vlen_enc = vector_length_encoding(this);
20219 __ vmulps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20220 %}
20221 ins_pipe( pipe_slow );
20222 %}
20223
20224 // Doubles vector mul
20225 instruct vmulD(vec dst, vec src) %{
20226 predicate(UseAVX == 0);
20227 match(Set dst (MulVD dst src));
20228 format %{ "mulpd $dst,$src\t! mul packedD" %}
20229 ins_encode %{
20230 __ mulpd($dst$$XMMRegister, $src$$XMMRegister);
20231 %}
20232 ins_pipe( pipe_slow );
20233 %}
20234
20235 instruct vmulD_reg(vec dst, vec src1, vec src2) %{
20236 predicate(UseAVX > 0);
20237 match(Set dst (MulVD src1 src2));
20238 format %{ "vmulpd $dst,$src1,$src2\t! mul packedD" %}
20239 ins_encode %{
20240 int vlen_enc = vector_length_encoding(this);
20241 __ vmulpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20242 %}
20243 ins_pipe( pipe_slow );
20244 %}
20245
20246 instruct vmulD_mem(vec dst, vec src, memory mem) %{
20247 predicate((UseAVX > 0) &&
20248 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20249 match(Set dst (MulVD src (LoadVector mem)));
20250 format %{ "vmulpd $dst,$src,$mem\t! mul packedD" %}
20251 ins_encode %{
20252 int vlen_enc = vector_length_encoding(this);
20253 __ vmulpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20254 %}
20255 ins_pipe( pipe_slow );
20256 %}
20257
20258 // --------------------------------- DIV --------------------------------------
20259
20260 // Floats vector div
20261 instruct vdivF(vec dst, vec src) %{
20262 predicate(UseAVX == 0);
20263 match(Set dst (DivVF dst src));
20264 format %{ "divps $dst,$src\t! div packedF" %}
20265 ins_encode %{
20266 __ divps($dst$$XMMRegister, $src$$XMMRegister);
20267 %}
20268 ins_pipe( pipe_slow );
20269 %}
20270
20271 instruct vdivF_reg(vec dst, vec src1, vec src2) %{
20272 predicate(UseAVX > 0);
20273 match(Set dst (DivVF src1 src2));
20274 format %{ "vdivps $dst,$src1,$src2\t! div packedF" %}
20275 ins_encode %{
20276 int vlen_enc = vector_length_encoding(this);
20277 __ vdivps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20278 %}
20279 ins_pipe( pipe_slow );
20280 %}
20281
20282 instruct vdivF_mem(vec dst, vec src, memory mem) %{
20283 predicate((UseAVX > 0) &&
20284 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20285 match(Set dst (DivVF src (LoadVector mem)));
20286 format %{ "vdivps $dst,$src,$mem\t! div packedF" %}
20287 ins_encode %{
20288 int vlen_enc = vector_length_encoding(this);
20289 __ vdivps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20290 %}
20291 ins_pipe( pipe_slow );
20292 %}
20293
20294 // Doubles vector div
20295 instruct vdivD(vec dst, vec src) %{
20296 predicate(UseAVX == 0);
20297 match(Set dst (DivVD dst src));
20298 format %{ "divpd $dst,$src\t! div packedD" %}
20299 ins_encode %{
20300 __ divpd($dst$$XMMRegister, $src$$XMMRegister);
20301 %}
20302 ins_pipe( pipe_slow );
20303 %}
20304
20305 instruct vdivD_reg(vec dst, vec src1, vec src2) %{
20306 predicate(UseAVX > 0);
20307 match(Set dst (DivVD src1 src2));
20308 format %{ "vdivpd $dst,$src1,$src2\t! div packedD" %}
20309 ins_encode %{
20310 int vlen_enc = vector_length_encoding(this);
20311 __ vdivpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20312 %}
20313 ins_pipe( pipe_slow );
20314 %}
20315
20316 instruct vdivD_mem(vec dst, vec src, memory mem) %{
20317 predicate((UseAVX > 0) &&
20318 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20319 match(Set dst (DivVD src (LoadVector mem)));
20320 format %{ "vdivpd $dst,$src,$mem\t! div packedD" %}
20321 ins_encode %{
20322 int vlen_enc = vector_length_encoding(this);
20323 __ vdivpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20324 %}
20325 ins_pipe( pipe_slow );
20326 %}
20327
20328 // ------------------------------ MinMax ---------------------------------------
20329
20330 // Byte, Short, Int vector Min/Max
20331 instruct minmax_reg_sse(vec dst, vec src) %{
20332 predicate(is_integral_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_element_basic_type(n) != T_LONG && // T_BYTE, T_SHORT, T_INT
20333 UseAVX == 0);
20334 match(Set dst (MinV dst src));
20335 match(Set dst (MaxV dst src));
20336 format %{ "vector_minmax $dst,$src\t! " %}
20337 ins_encode %{
20338 assert(UseSSE >= 4, "required");
20339
20340 int opcode = this->ideal_Opcode();
20341 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20342 __ pminmax(opcode, elem_bt, $dst$$XMMRegister, $src$$XMMRegister);
20343 %}
20344 ins_pipe( pipe_slow );
20345 %}
20346
20347 instruct vminmax_reg(vec dst, vec src1, vec src2) %{
20348 predicate(is_integral_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_element_basic_type(n) != T_LONG && // T_BYTE, T_SHORT, T_INT
20349 UseAVX > 0);
20350 match(Set dst (MinV src1 src2));
20351 match(Set dst (MaxV src1 src2));
20352 format %{ "vector_minmax $dst,$src1,$src2\t! " %}
20353 ins_encode %{
20354 int opcode = this->ideal_Opcode();
20355 int vlen_enc = vector_length_encoding(this);
20356 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20357
20358 __ vpminmax(opcode, elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20359 %}
20360 ins_pipe( pipe_slow );
20361 %}
20362
20363 // Long vector Min/Max
20364 instruct minmaxL_reg_sse(vec dst, vec src, rxmm0 tmp) %{
20365 predicate(Matcher::vector_length_in_bytes(n) == 16 && Matcher::vector_element_basic_type(n) == T_LONG &&
20366 UseAVX == 0);
20367 match(Set dst (MinV dst src));
20368 match(Set dst (MaxV src dst));
20369 effect(TEMP dst, TEMP tmp);
20370 format %{ "vector_minmaxL $dst,$src\t!using $tmp as TEMP" %}
20371 ins_encode %{
20372 assert(UseSSE >= 4, "required");
20373
20374 int opcode = this->ideal_Opcode();
20375 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20376 assert(elem_bt == T_LONG, "sanity");
20377
20378 __ pminmax(opcode, elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $tmp$$XMMRegister);
20379 %}
20380 ins_pipe( pipe_slow );
20381 %}
20382
20383 instruct vminmaxL_reg_avx(legVec dst, legVec src1, legVec src2) %{
20384 predicate(Matcher::vector_length_in_bytes(n) <= 32 && Matcher::vector_element_basic_type(n) == T_LONG &&
20385 UseAVX > 0 && !VM_Version::supports_avx512vl());
20386 match(Set dst (MinV src1 src2));
20387 match(Set dst (MaxV src1 src2));
20388 effect(TEMP dst);
20389 format %{ "vector_minmaxL $dst,$src1,$src2\t! " %}
20390 ins_encode %{
20391 int vlen_enc = vector_length_encoding(this);
20392 int opcode = this->ideal_Opcode();
20393 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20394 assert(elem_bt == T_LONG, "sanity");
20395
20396 __ vpminmax(opcode, elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20397 %}
20398 ins_pipe( pipe_slow );
20399 %}
20400
20401 instruct vminmaxL_reg_evex(vec dst, vec src1, vec src2) %{
20402 predicate((Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()) &&
20403 Matcher::vector_element_basic_type(n) == T_LONG);
20404 match(Set dst (MinV src1 src2));
20405 match(Set dst (MaxV src1 src2));
20406 format %{ "vector_minmaxL $dst,$src1,src2\t! " %}
20407 ins_encode %{
20408 assert(UseAVX > 2, "required");
20409
20410 int vlen_enc = vector_length_encoding(this);
20411 int opcode = this->ideal_Opcode();
20412 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20413 assert(elem_bt == T_LONG, "sanity");
20414
20415 __ vpminmax(opcode, elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20416 %}
20417 ins_pipe( pipe_slow );
20418 %}
20419
20420 // Float/Double vector Min/Max
20421 instruct minmaxFP_reg_avx10_2(vec dst, vec a, vec b) %{
20422 predicate(VM_Version::supports_avx10_2() &&
20423 is_floating_point_type(Matcher::vector_element_basic_type(n))); // T_FLOAT, T_DOUBLE
20424 match(Set dst (MinV a b));
20425 match(Set dst (MaxV a b));
20426 format %{ "vector_minmaxFP $dst, $a, $b" %}
20427 ins_encode %{
20428 int vlen_enc = vector_length_encoding(this);
20429 int opcode = this->ideal_Opcode();
20430 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20431 __ vminmax_fp_avx10_2(opcode, elem_bt, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister, vlen_enc);
20432 %}
20433 ins_pipe( pipe_slow );
20434 %}
20435
20436 // Float/Double vector Min/Max
20437 instruct minmaxFP_reg(legVec dst, legVec a, legVec b, legVec tmp, legVec atmp, legVec btmp) %{
20438 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_length_in_bytes(n) <= 32 &&
20439 is_floating_point_type(Matcher::vector_element_basic_type(n)) && // T_FLOAT, T_DOUBLE
20440 UseAVX > 0);
20441 match(Set dst (MinV a b));
20442 match(Set dst (MaxV a b));
20443 effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
20444 format %{ "vector_minmaxFP $dst,$a,$b\t!using $tmp, $atmp, $btmp as TEMP" %}
20445 ins_encode %{
20446 assert(UseAVX > 0, "required");
20447
20448 int opcode = this->ideal_Opcode();
20449 int vlen_enc = vector_length_encoding(this);
20450 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20451
20452 __ vminmax_fp(opcode, elem_bt,
20453 $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister,
20454 $tmp$$XMMRegister, $atmp$$XMMRegister , $btmp$$XMMRegister, vlen_enc);
20455 %}
20456 ins_pipe( pipe_slow );
20457 %}
20458
20459 instruct evminmaxFP_reg_evex(vec dst, vec a, vec b, vec atmp, vec btmp, kReg ktmp) %{
20460 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_length_in_bytes(n) == 64 &&
20461 is_floating_point_type(Matcher::vector_element_basic_type(n))); // T_FLOAT, T_DOUBLE
20462 match(Set dst (MinV a b));
20463 match(Set dst (MaxV a b));
20464 effect(TEMP dst, USE a, USE b, TEMP atmp, TEMP btmp, TEMP ktmp);
20465 format %{ "vector_minmaxFP $dst,$a,$b\t!using $atmp, $btmp as TEMP" %}
20466 ins_encode %{
20467 assert(UseAVX > 2, "required");
20468
20469 int opcode = this->ideal_Opcode();
20470 int vlen_enc = vector_length_encoding(this);
20471 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20472
20473 __ evminmax_fp(opcode, elem_bt,
20474 $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister,
20475 $ktmp$$KRegister, $atmp$$XMMRegister , $btmp$$XMMRegister, vlen_enc);
20476 %}
20477 ins_pipe( pipe_slow );
20478 %}
20479
20480 // ------------------------------ Unsigned vector Min/Max ----------------------
20481
20482 instruct vector_uminmax_reg(vec dst, vec a, vec b) %{
20483 predicate(VM_Version::supports_avx512vl() || Matcher::vector_element_basic_type(n) != T_LONG);
20484 match(Set dst (UMinV a b));
20485 match(Set dst (UMaxV a b));
20486 format %{ "vector_uminmax $dst,$a,$b\t!" %}
20487 ins_encode %{
20488 int opcode = this->ideal_Opcode();
20489 int vlen_enc = vector_length_encoding(this);
20490 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20491 assert(is_integral_type(elem_bt), "");
20492 __ vpuminmax(opcode, elem_bt, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vlen_enc);
20493 %}
20494 ins_pipe( pipe_slow );
20495 %}
20496
20497 instruct vector_uminmax_mem(vec dst, vec a, memory b) %{
20498 predicate(VM_Version::supports_avx512vl() || Matcher::vector_element_basic_type(n) != T_LONG);
20499 match(Set dst (UMinV a (LoadVector b)));
20500 match(Set dst (UMaxV a (LoadVector b)));
20501 format %{ "vector_uminmax $dst,$a,$b\t!" %}
20502 ins_encode %{
20503 int opcode = this->ideal_Opcode();
20504 int vlen_enc = vector_length_encoding(this);
20505 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20506 assert(is_integral_type(elem_bt), "");
20507 __ vpuminmax(opcode, elem_bt, $dst$$XMMRegister, $a$$XMMRegister, $b$$Address, vlen_enc);
20508 %}
20509 ins_pipe( pipe_slow );
20510 %}
20511
20512 instruct vector_uminmaxq_reg(vec dst, vec a, vec b, vec xtmp1, vec xtmp2) %{
20513 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_element_basic_type(n) == T_LONG);
20514 match(Set dst (UMinV a b));
20515 match(Set dst (UMaxV a b));
20516 effect(TEMP xtmp1, TEMP xtmp2);
20517 format %{ "vector_uminmaxq $dst,$a,$b\t! using xtmp1 and xtmp2 as TEMP" %}
20518 ins_encode %{
20519 int opcode = this->ideal_Opcode();
20520 int vlen_enc = vector_length_encoding(this);
20521 __ vpuminmaxq(opcode, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20522 %}
20523 ins_pipe( pipe_slow );
20524 %}
20525
20526 instruct vector_uminmax_reg_masked(vec dst, vec src2, kReg mask) %{
20527 match(Set dst (UMinV (Binary dst src2) mask));
20528 match(Set dst (UMaxV (Binary dst src2) mask));
20529 format %{ "vector_uminmax_masked $dst, $dst, $src2, $mask\t! umin/max masked operation" %}
20530 ins_encode %{
20531 int vlen_enc = vector_length_encoding(this);
20532 BasicType bt = Matcher::vector_element_basic_type(this);
20533 int opc = this->ideal_Opcode();
20534 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
20535 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
20536 %}
20537 ins_pipe( pipe_slow );
20538 %}
20539
20540 instruct vector_uminmax_mem_masked(vec dst, memory src2, kReg mask) %{
20541 match(Set dst (UMinV (Binary dst (LoadVector src2)) mask));
20542 match(Set dst (UMaxV (Binary dst (LoadVector src2)) mask));
20543 format %{ "vector_uminmax_masked $dst, $dst, $src2, $mask\t! umin/max masked operation" %}
20544 ins_encode %{
20545 int vlen_enc = vector_length_encoding(this);
20546 BasicType bt = Matcher::vector_element_basic_type(this);
20547 int opc = this->ideal_Opcode();
20548 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
20549 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
20550 %}
20551 ins_pipe( pipe_slow );
20552 %}
20553
20554 // --------------------------------- Signum/CopySign ---------------------------
20555
20556 instruct signumF_reg(regF dst, regF zero, regF one, rFlagsReg cr) %{
20557 match(Set dst (SignumF dst (Binary zero one)));
20558 effect(KILL cr);
20559 format %{ "signumF $dst, $dst" %}
20560 ins_encode %{
20561 int opcode = this->ideal_Opcode();
20562 __ signum_fp(opcode, $dst$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister);
20563 %}
20564 ins_pipe( pipe_slow );
20565 %}
20566
20567 instruct signumD_reg(regD dst, regD zero, regD one, rFlagsReg cr) %{
20568 match(Set dst (SignumD dst (Binary zero one)));
20569 effect(KILL cr);
20570 format %{ "signumD $dst, $dst" %}
20571 ins_encode %{
20572 int opcode = this->ideal_Opcode();
20573 __ signum_fp(opcode, $dst$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister);
20574 %}
20575 ins_pipe( pipe_slow );
20576 %}
20577
20578 instruct signumV_reg_avx(vec dst, vec src, vec zero, vec one, vec xtmp1) %{
20579 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n) <= 32);
20580 match(Set dst (SignumVF src (Binary zero one)));
20581 match(Set dst (SignumVD src (Binary zero one)));
20582 effect(TEMP dst, TEMP xtmp1);
20583 format %{ "vector_signum_avx $dst, $src\t! using $xtmp1 as TEMP" %}
20584 ins_encode %{
20585 int opcode = this->ideal_Opcode();
20586 int vec_enc = vector_length_encoding(this);
20587 __ vector_signum_avx(opcode, $dst$$XMMRegister, $src$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister,
20588 $xtmp1$$XMMRegister, vec_enc);
20589 %}
20590 ins_pipe( pipe_slow );
20591 %}
20592
20593 instruct signumV_reg_evex(vec dst, vec src, vec zero, vec one, kReg ktmp1) %{
20594 predicate(VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64);
20595 match(Set dst (SignumVF src (Binary zero one)));
20596 match(Set dst (SignumVD src (Binary zero one)));
20597 effect(TEMP dst, TEMP ktmp1);
20598 format %{ "vector_signum_evex $dst, $src\t! using $ktmp1 as TEMP" %}
20599 ins_encode %{
20600 int opcode = this->ideal_Opcode();
20601 int vec_enc = vector_length_encoding(this);
20602 __ vector_signum_evex(opcode, $dst$$XMMRegister, $src$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister,
20603 $ktmp1$$KRegister, vec_enc);
20604 %}
20605 ins_pipe( pipe_slow );
20606 %}
20607
20608 // ---------------------------------------
20609 // For copySign use 0xE4 as writemask for vpternlog
20610 // Desired Truth Table: A -> xmm0 bit, B -> xmm1 bit, C -> xmm2 bit
20611 // C (xmm2) is set to 0x7FFFFFFF
20612 // Wherever xmm2 is 0, we want to pick from B (sign)
20613 // Wherever xmm2 is 1, we want to pick from A (src)
20614 //
20615 // A B C Result
20616 // 0 0 0 0
20617 // 0 0 1 0
20618 // 0 1 0 1
20619 // 0 1 1 0
20620 // 1 0 0 0
20621 // 1 0 1 1
20622 // 1 1 0 1
20623 // 1 1 1 1
20624 //
20625 // Result going from high bit to low bit is 0x11100100 = 0xe4
20626 // ---------------------------------------
20627
20628 instruct copySignF_reg(regF dst, regF src, regF tmp1, rRegI tmp2) %{
20629 match(Set dst (CopySignF dst src));
20630 effect(TEMP tmp1, TEMP tmp2);
20631 format %{ "CopySignF $dst, $src\t! using $tmp1 and $tmp2 as TEMP" %}
20632 ins_encode %{
20633 __ movl($tmp2$$Register, 0x7FFFFFFF);
20634 __ movdl($tmp1$$XMMRegister, $tmp2$$Register);
20635 __ vpternlogd($dst$$XMMRegister, 0xE4, $src$$XMMRegister, $tmp1$$XMMRegister, Assembler::AVX_128bit);
20636 %}
20637 ins_pipe( pipe_slow );
20638 %}
20639
20640 instruct copySignD_imm(regD dst, regD src, regD tmp1, rRegL tmp2, immD zero) %{
20641 match(Set dst (CopySignD dst (Binary src zero)));
20642 ins_cost(100);
20643 effect(TEMP tmp1, TEMP tmp2);
20644 format %{ "CopySignD $dst, $src\t! using $tmp1 and $tmp2 as TEMP" %}
20645 ins_encode %{
20646 __ mov64($tmp2$$Register, 0x7FFFFFFFFFFFFFFF);
20647 __ movq($tmp1$$XMMRegister, $tmp2$$Register);
20648 __ vpternlogq($dst$$XMMRegister, 0xE4, $src$$XMMRegister, $tmp1$$XMMRegister, Assembler::AVX_128bit);
20649 %}
20650 ins_pipe( pipe_slow );
20651 %}
20652
20653 //----------------------------- CompressBits/ExpandBits ------------------------
20654
20655 instruct compressBitsI_reg(rRegI dst, rRegI src, rRegI mask) %{
20656 predicate(n->bottom_type()->isa_int());
20657 match(Set dst (CompressBits src mask));
20658 format %{ "pextl $dst, $src, $mask\t! parallel bit extract" %}
20659 ins_encode %{
20660 __ pextl($dst$$Register, $src$$Register, $mask$$Register);
20661 %}
20662 ins_pipe( pipe_slow );
20663 %}
20664
20665 instruct expandBitsI_reg(rRegI dst, rRegI src, rRegI mask) %{
20666 predicate(n->bottom_type()->isa_int());
20667 match(Set dst (ExpandBits src mask));
20668 format %{ "pdepl $dst, $src, $mask\t! parallel bit deposit" %}
20669 ins_encode %{
20670 __ pdepl($dst$$Register, $src$$Register, $mask$$Register);
20671 %}
20672 ins_pipe( pipe_slow );
20673 %}
20674
20675 instruct compressBitsI_mem(rRegI dst, rRegI src, memory mask) %{
20676 predicate(n->bottom_type()->isa_int());
20677 match(Set dst (CompressBits src (LoadI mask)));
20678 format %{ "pextl $dst, $src, $mask\t! parallel bit extract" %}
20679 ins_encode %{
20680 __ pextl($dst$$Register, $src$$Register, $mask$$Address);
20681 %}
20682 ins_pipe( pipe_slow );
20683 %}
20684
20685 instruct expandBitsI_mem(rRegI dst, rRegI src, memory mask) %{
20686 predicate(n->bottom_type()->isa_int());
20687 match(Set dst (ExpandBits src (LoadI mask)));
20688 format %{ "pdepl $dst, $src, $mask\t! parallel bit deposit" %}
20689 ins_encode %{
20690 __ pdepl($dst$$Register, $src$$Register, $mask$$Address);
20691 %}
20692 ins_pipe( pipe_slow );
20693 %}
20694
20695 // --------------------------------- Sqrt --------------------------------------
20696
20697 instruct vsqrtF_reg(vec dst, vec src) %{
20698 match(Set dst (SqrtVF src));
20699 format %{ "vsqrtps $dst,$src\t! sqrt packedF" %}
20700 ins_encode %{
20701 assert(UseAVX > 0, "required");
20702 int vlen_enc = vector_length_encoding(this);
20703 __ vsqrtps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
20704 %}
20705 ins_pipe( pipe_slow );
20706 %}
20707
20708 instruct vsqrtF_mem(vec dst, memory mem) %{
20709 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
20710 match(Set dst (SqrtVF (LoadVector mem)));
20711 format %{ "vsqrtps $dst,$mem\t! sqrt packedF" %}
20712 ins_encode %{
20713 assert(UseAVX > 0, "required");
20714 int vlen_enc = vector_length_encoding(this);
20715 __ vsqrtps($dst$$XMMRegister, $mem$$Address, vlen_enc);
20716 %}
20717 ins_pipe( pipe_slow );
20718 %}
20719
20720 // Floating point vector sqrt
20721 instruct vsqrtD_reg(vec dst, vec src) %{
20722 match(Set dst (SqrtVD src));
20723 format %{ "vsqrtpd $dst,$src\t! sqrt packedD" %}
20724 ins_encode %{
20725 assert(UseAVX > 0, "required");
20726 int vlen_enc = vector_length_encoding(this);
20727 __ vsqrtpd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
20728 %}
20729 ins_pipe( pipe_slow );
20730 %}
20731
20732 instruct vsqrtD_mem(vec dst, memory mem) %{
20733 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
20734 match(Set dst (SqrtVD (LoadVector mem)));
20735 format %{ "vsqrtpd $dst,$mem\t! sqrt packedD" %}
20736 ins_encode %{
20737 assert(UseAVX > 0, "required");
20738 int vlen_enc = vector_length_encoding(this);
20739 __ vsqrtpd($dst$$XMMRegister, $mem$$Address, vlen_enc);
20740 %}
20741 ins_pipe( pipe_slow );
20742 %}
20743
20744 // ------------------------------ Shift ---------------------------------------
20745
20746 // Left and right shift count vectors are the same on x86
20747 // (only lowest bits of xmm reg are used for count).
20748 instruct vshiftcnt(vec dst, rRegI cnt) %{
20749 match(Set dst (LShiftCntV cnt));
20750 match(Set dst (RShiftCntV cnt));
20751 format %{ "movdl $dst,$cnt\t! load shift count" %}
20752 ins_encode %{
20753 __ movdl($dst$$XMMRegister, $cnt$$Register);
20754 %}
20755 ins_pipe( pipe_slow );
20756 %}
20757
20758 // Byte vector shift
20759 instruct vshiftB(vec dst, vec src, vec shift, vec tmp) %{
20760 predicate(Matcher::vector_length(n) <= 8 && !n->as_ShiftV()->is_var_shift());
20761 match(Set dst ( LShiftVB src shift));
20762 match(Set dst ( RShiftVB src shift));
20763 match(Set dst (URShiftVB src shift));
20764 effect(TEMP dst, USE src, USE shift, TEMP tmp);
20765 format %{"vector_byte_shift $dst,$src,$shift" %}
20766 ins_encode %{
20767 assert(UseSSE > 3, "required");
20768 int opcode = this->ideal_Opcode();
20769 bool sign = (opcode != Op_URShiftVB);
20770 __ vextendbw(sign, $tmp$$XMMRegister, $src$$XMMRegister);
20771 __ vshiftw(opcode, $tmp$$XMMRegister, $shift$$XMMRegister);
20772 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), noreg);
20773 __ pand($dst$$XMMRegister, $tmp$$XMMRegister);
20774 __ packuswb($dst$$XMMRegister, $dst$$XMMRegister);
20775 %}
20776 ins_pipe( pipe_slow );
20777 %}
20778
20779 instruct vshift16B(vec dst, vec src, vec shift, vec tmp1, vec tmp2) %{
20780 predicate(Matcher::vector_length(n) == 16 && !n->as_ShiftV()->is_var_shift() &&
20781 UseAVX <= 1);
20782 match(Set dst ( LShiftVB src shift));
20783 match(Set dst ( RShiftVB src shift));
20784 match(Set dst (URShiftVB src shift));
20785 effect(TEMP dst, USE src, USE shift, TEMP tmp1, TEMP tmp2);
20786 format %{"vector_byte_shift $dst,$src,$shift" %}
20787 ins_encode %{
20788 assert(UseSSE > 3, "required");
20789 int opcode = this->ideal_Opcode();
20790 bool sign = (opcode != Op_URShiftVB);
20791 __ vextendbw(sign, $tmp1$$XMMRegister, $src$$XMMRegister);
20792 __ vshiftw(opcode, $tmp1$$XMMRegister, $shift$$XMMRegister);
20793 __ pshufd($tmp2$$XMMRegister, $src$$XMMRegister, 0xE);
20794 __ vextendbw(sign, $tmp2$$XMMRegister, $tmp2$$XMMRegister);
20795 __ vshiftw(opcode, $tmp2$$XMMRegister, $shift$$XMMRegister);
20796 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), noreg);
20797 __ pand($tmp2$$XMMRegister, $dst$$XMMRegister);
20798 __ pand($dst$$XMMRegister, $tmp1$$XMMRegister);
20799 __ packuswb($dst$$XMMRegister, $tmp2$$XMMRegister);
20800 %}
20801 ins_pipe( pipe_slow );
20802 %}
20803
20804 instruct vshift16B_avx(vec dst, vec src, vec shift, vec tmp) %{
20805 predicate(Matcher::vector_length(n) == 16 && !n->as_ShiftV()->is_var_shift() &&
20806 UseAVX > 1);
20807 match(Set dst ( LShiftVB src shift));
20808 match(Set dst ( RShiftVB src shift));
20809 match(Set dst (URShiftVB src shift));
20810 effect(TEMP dst, TEMP tmp);
20811 format %{"vector_byte_shift $dst,$src,$shift" %}
20812 ins_encode %{
20813 int opcode = this->ideal_Opcode();
20814 bool sign = (opcode != Op_URShiftVB);
20815 int vlen_enc = Assembler::AVX_256bit;
20816 __ vextendbw(sign, $tmp$$XMMRegister, $src$$XMMRegister, vlen_enc);
20817 __ vshiftw(opcode, $tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20818 __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
20819 __ vextracti128_high($dst$$XMMRegister, $tmp$$XMMRegister);
20820 __ vpackuswb($dst$$XMMRegister, $tmp$$XMMRegister, $dst$$XMMRegister, 0);
20821 %}
20822 ins_pipe( pipe_slow );
20823 %}
20824
20825 instruct vshift32B_avx(vec dst, vec src, vec shift, vec tmp) %{
20826 predicate(Matcher::vector_length(n) == 32 && !n->as_ShiftV()->is_var_shift());
20827 match(Set dst ( LShiftVB src shift));
20828 match(Set dst ( RShiftVB src shift));
20829 match(Set dst (URShiftVB src shift));
20830 effect(TEMP dst, TEMP tmp);
20831 format %{"vector_byte_shift $dst,$src,$shift" %}
20832 ins_encode %{
20833 assert(UseAVX > 1, "required");
20834 int opcode = this->ideal_Opcode();
20835 bool sign = (opcode != Op_URShiftVB);
20836 int vlen_enc = Assembler::AVX_256bit;
20837 __ vextracti128_high($tmp$$XMMRegister, $src$$XMMRegister);
20838 __ vextendbw(sign, $tmp$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
20839 __ vextendbw(sign, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
20840 __ vshiftw(opcode, $tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20841 __ vshiftw(opcode, $dst$$XMMRegister, $dst$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20842 __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
20843 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
20844 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
20845 __ vpermq($dst$$XMMRegister, $dst$$XMMRegister, 0xD8, vlen_enc);
20846 %}
20847 ins_pipe( pipe_slow );
20848 %}
20849
20850 instruct vshift64B_avx(vec dst, vec src, vec shift, vec tmp1, vec tmp2) %{
20851 predicate(Matcher::vector_length(n) == 64 && !n->as_ShiftV()->is_var_shift());
20852 match(Set dst ( LShiftVB src shift));
20853 match(Set dst (RShiftVB src shift));
20854 match(Set dst (URShiftVB src shift));
20855 effect(TEMP dst, TEMP tmp1, TEMP tmp2);
20856 format %{"vector_byte_shift $dst,$src,$shift" %}
20857 ins_encode %{
20858 assert(UseAVX > 2, "required");
20859 int opcode = this->ideal_Opcode();
20860 bool sign = (opcode != Op_URShiftVB);
20861 int vlen_enc = Assembler::AVX_512bit;
20862 __ vextracti64x4($tmp1$$XMMRegister, $src$$XMMRegister, 1);
20863 __ vextendbw(sign, $tmp1$$XMMRegister, $tmp1$$XMMRegister, vlen_enc);
20864 __ vextendbw(sign, $tmp2$$XMMRegister, $src$$XMMRegister, vlen_enc);
20865 __ vshiftw(opcode, $tmp1$$XMMRegister, $tmp1$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20866 __ vshiftw(opcode, $tmp2$$XMMRegister, $tmp2$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20867 __ vmovdqu($dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), noreg);
20868 __ vpbroadcastd($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
20869 __ vpand($tmp1$$XMMRegister, $tmp1$$XMMRegister, $dst$$XMMRegister, vlen_enc);
20870 __ vpand($tmp2$$XMMRegister, $tmp2$$XMMRegister, $dst$$XMMRegister, vlen_enc);
20871 __ vpackuswb($dst$$XMMRegister, $tmp1$$XMMRegister, $tmp2$$XMMRegister, vlen_enc);
20872 __ evmovdquq($tmp2$$XMMRegister, ExternalAddress(vector_byte_perm_mask()), vlen_enc, noreg);
20873 __ vpermq($dst$$XMMRegister, $tmp2$$XMMRegister, $dst$$XMMRegister, vlen_enc);
20874 %}
20875 ins_pipe( pipe_slow );
20876 %}
20877
20878 // Shorts vector logical right shift produces incorrect Java result
20879 // for negative data because java code convert short value into int with
20880 // sign extension before a shift. But char vectors are fine since chars are
20881 // unsigned values.
20882 // Shorts/Chars vector left shift
20883 instruct vshiftS(vec dst, vec src, vec shift) %{
20884 predicate(!n->as_ShiftV()->is_var_shift());
20885 match(Set dst ( LShiftVS src shift));
20886 match(Set dst ( RShiftVS src shift));
20887 match(Set dst (URShiftVS src shift));
20888 effect(TEMP dst, USE src, USE shift);
20889 format %{ "vshiftw $dst,$src,$shift\t! shift packedS" %}
20890 ins_encode %{
20891 int opcode = this->ideal_Opcode();
20892 if (UseAVX > 0) {
20893 int vlen_enc = vector_length_encoding(this);
20894 __ vshiftw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20895 } else {
20896 int vlen = Matcher::vector_length(this);
20897 if (vlen == 2) {
20898 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
20899 __ vshiftw(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
20900 } else if (vlen == 4) {
20901 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
20902 __ vshiftw(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
20903 } else {
20904 assert (vlen == 8, "sanity");
20905 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
20906 __ vshiftw(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
20907 }
20908 }
20909 %}
20910 ins_pipe( pipe_slow );
20911 %}
20912
20913 // Integers vector left shift
20914 instruct vshiftI(vec dst, vec src, vec shift) %{
20915 predicate(!n->as_ShiftV()->is_var_shift());
20916 match(Set dst ( LShiftVI src shift));
20917 match(Set dst ( RShiftVI src shift));
20918 match(Set dst (URShiftVI src shift));
20919 effect(TEMP dst, USE src, USE shift);
20920 format %{ "vshiftd $dst,$src,$shift\t! shift packedI" %}
20921 ins_encode %{
20922 int opcode = this->ideal_Opcode();
20923 if (UseAVX > 0) {
20924 int vlen_enc = vector_length_encoding(this);
20925 __ vshiftd(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20926 } else {
20927 int vlen = Matcher::vector_length(this);
20928 if (vlen == 2) {
20929 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
20930 __ vshiftd(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
20931 } else {
20932 assert(vlen == 4, "sanity");
20933 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
20934 __ vshiftd(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
20935 }
20936 }
20937 %}
20938 ins_pipe( pipe_slow );
20939 %}
20940
20941 // Integers vector left constant shift
20942 instruct vshiftI_imm(vec dst, vec src, immI8 shift) %{
20943 match(Set dst (LShiftVI src (LShiftCntV shift)));
20944 match(Set dst (RShiftVI src (RShiftCntV shift)));
20945 match(Set dst (URShiftVI src (RShiftCntV shift)));
20946 format %{ "vshiftd_imm $dst,$src,$shift\t! shift packedI" %}
20947 ins_encode %{
20948 int opcode = this->ideal_Opcode();
20949 if (UseAVX > 0) {
20950 int vector_len = vector_length_encoding(this);
20951 __ vshiftd_imm(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$constant, vector_len);
20952 } else {
20953 int vlen = Matcher::vector_length(this);
20954 if (vlen == 2) {
20955 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
20956 __ vshiftd_imm(opcode, $dst$$XMMRegister, $shift$$constant);
20957 } else {
20958 assert(vlen == 4, "sanity");
20959 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
20960 __ vshiftd_imm(opcode, $dst$$XMMRegister, $shift$$constant);
20961 }
20962 }
20963 %}
20964 ins_pipe( pipe_slow );
20965 %}
20966
20967 // Longs vector shift
20968 instruct vshiftL(vec dst, vec src, vec shift) %{
20969 predicate(!n->as_ShiftV()->is_var_shift());
20970 match(Set dst ( LShiftVL src shift));
20971 match(Set dst (URShiftVL src shift));
20972 effect(TEMP dst, USE src, USE shift);
20973 format %{ "vshiftq $dst,$src,$shift\t! shift packedL" %}
20974 ins_encode %{
20975 int opcode = this->ideal_Opcode();
20976 if (UseAVX > 0) {
20977 int vlen_enc = vector_length_encoding(this);
20978 __ vshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20979 } else {
20980 assert(Matcher::vector_length(this) == 2, "");
20981 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
20982 __ vshiftq(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
20983 }
20984 %}
20985 ins_pipe( pipe_slow );
20986 %}
20987
20988 // Longs vector constant shift
20989 instruct vshiftL_imm(vec dst, vec src, immI8 shift) %{
20990 match(Set dst (LShiftVL src (LShiftCntV shift)));
20991 match(Set dst (URShiftVL src (RShiftCntV shift)));
20992 format %{ "vshiftq_imm $dst,$src,$shift\t! shift packedL" %}
20993 ins_encode %{
20994 int opcode = this->ideal_Opcode();
20995 if (UseAVX > 0) {
20996 int vector_len = vector_length_encoding(this);
20997 __ vshiftq_imm(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$constant, vector_len);
20998 } else {
20999 assert(Matcher::vector_length(this) == 2, "");
21000 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
21001 __ vshiftq_imm(opcode, $dst$$XMMRegister, $shift$$constant);
21002 }
21003 %}
21004 ins_pipe( pipe_slow );
21005 %}
21006
21007 // -------------------ArithmeticRightShift -----------------------------------
21008 // Long vector arithmetic right shift
21009 instruct vshiftL_arith_reg(vec dst, vec src, vec shift, vec tmp) %{
21010 predicate(!n->as_ShiftV()->is_var_shift() && UseAVX <= 2);
21011 match(Set dst (RShiftVL src shift));
21012 effect(TEMP dst, TEMP tmp);
21013 format %{ "vshiftq $dst,$src,$shift" %}
21014 ins_encode %{
21015 uint vlen = Matcher::vector_length(this);
21016 if (vlen == 2) {
21017 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
21018 __ psrlq($dst$$XMMRegister, $shift$$XMMRegister);
21019 __ movdqu($tmp$$XMMRegister, ExternalAddress(vector_long_sign_mask()), noreg);
21020 __ psrlq($tmp$$XMMRegister, $shift$$XMMRegister);
21021 __ pxor($dst$$XMMRegister, $tmp$$XMMRegister);
21022 __ psubq($dst$$XMMRegister, $tmp$$XMMRegister);
21023 } else {
21024 assert(vlen == 4, "sanity");
21025 assert(UseAVX > 1, "required");
21026 int vlen_enc = Assembler::AVX_256bit;
21027 __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21028 __ vmovdqu($tmp$$XMMRegister, ExternalAddress(vector_long_sign_mask()), noreg);
21029 __ vpsrlq($tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21030 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
21031 __ vpsubq($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
21032 }
21033 %}
21034 ins_pipe( pipe_slow );
21035 %}
21036
21037 instruct vshiftL_arith_reg_evex(vec dst, vec src, vec shift) %{
21038 predicate(!n->as_ShiftV()->is_var_shift() && UseAVX > 2);
21039 match(Set dst (RShiftVL src shift));
21040 format %{ "vshiftq $dst,$src,$shift" %}
21041 ins_encode %{
21042 int vlen_enc = vector_length_encoding(this);
21043 __ evpsraq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21044 %}
21045 ins_pipe( pipe_slow );
21046 %}
21047
21048 // ------------------- Variable Shift -----------------------------
21049 // Byte variable shift
21050 instruct vshift8B_var_nobw(vec dst, vec src, vec shift, vec vtmp) %{
21051 predicate(Matcher::vector_length(n) <= 8 &&
21052 n->as_ShiftV()->is_var_shift() &&
21053 !VM_Version::supports_avx512bw());
21054 match(Set dst ( LShiftVB src shift));
21055 match(Set dst ( RShiftVB src shift));
21056 match(Set dst (URShiftVB src shift));
21057 effect(TEMP dst, TEMP vtmp);
21058 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp as TEMP" %}
21059 ins_encode %{
21060 assert(UseAVX >= 2, "required");
21061
21062 int opcode = this->ideal_Opcode();
21063 int vlen_enc = Assembler::AVX_128bit;
21064 __ varshiftbw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp$$XMMRegister);
21065 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, 0);
21066 %}
21067 ins_pipe( pipe_slow );
21068 %}
21069
21070 instruct vshift16B_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2) %{
21071 predicate(Matcher::vector_length(n) == 16 &&
21072 n->as_ShiftV()->is_var_shift() &&
21073 !VM_Version::supports_avx512bw());
21074 match(Set dst ( LShiftVB src shift));
21075 match(Set dst ( RShiftVB src shift));
21076 match(Set dst (URShiftVB src shift));
21077 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
21078 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp1, $vtmp2 as TEMP" %}
21079 ins_encode %{
21080 assert(UseAVX >= 2, "required");
21081
21082 int opcode = this->ideal_Opcode();
21083 int vlen_enc = Assembler::AVX_128bit;
21084 // Shift lower half and get word result in dst
21085 __ varshiftbw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp1$$XMMRegister);
21086
21087 // Shift upper half and get word result in vtmp1
21088 __ vpshufd($vtmp1$$XMMRegister, $src$$XMMRegister, 0xE, 0);
21089 __ vpshufd($vtmp2$$XMMRegister, $shift$$XMMRegister, 0xE, 0);
21090 __ varshiftbw(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21091
21092 // Merge and down convert the two word results to byte in dst
21093 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0);
21094 %}
21095 ins_pipe( pipe_slow );
21096 %}
21097
21098 instruct vshift32B_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2, vec vtmp3, vec vtmp4) %{
21099 predicate(Matcher::vector_length(n) == 32 &&
21100 n->as_ShiftV()->is_var_shift() &&
21101 !VM_Version::supports_avx512bw());
21102 match(Set dst ( LShiftVB src shift));
21103 match(Set dst ( RShiftVB src shift));
21104 match(Set dst (URShiftVB src shift));
21105 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4);
21106 format %{ "vector_varshift_byte $dst, $src, $shift\n\t using $vtmp1, $vtmp2, $vtmp3, $vtmp4 as TEMP" %}
21107 ins_encode %{
21108 assert(UseAVX >= 2, "required");
21109
21110 int opcode = this->ideal_Opcode();
21111 int vlen_enc = Assembler::AVX_128bit;
21112 // Process lower 128 bits and get result in dst
21113 __ varshiftbw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp1$$XMMRegister);
21114 __ vpshufd($vtmp1$$XMMRegister, $src$$XMMRegister, 0xE, 0);
21115 __ vpshufd($vtmp2$$XMMRegister, $shift$$XMMRegister, 0xE, 0);
21116 __ varshiftbw(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21117 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0);
21118
21119 // Process higher 128 bits and get result in vtmp3
21120 __ vextracti128_high($vtmp1$$XMMRegister, $src$$XMMRegister);
21121 __ vextracti128_high($vtmp2$$XMMRegister, $shift$$XMMRegister);
21122 __ varshiftbw(opcode, $vtmp3$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp4$$XMMRegister);
21123 __ vpshufd($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, 0xE, 0);
21124 __ vpshufd($vtmp2$$XMMRegister, $vtmp2$$XMMRegister, 0xE, 0);
21125 __ varshiftbw(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21126 __ vpackuswb($vtmp1$$XMMRegister, $vtmp3$$XMMRegister, $vtmp1$$XMMRegister, 0);
21127
21128 // Merge the two results in dst
21129 __ vinserti128($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0x1);
21130 %}
21131 ins_pipe( pipe_slow );
21132 %}
21133
21134 instruct vshiftB_var_evex_bw(vec dst, vec src, vec shift, vec vtmp) %{
21135 predicate(Matcher::vector_length(n) <= 32 &&
21136 n->as_ShiftV()->is_var_shift() &&
21137 VM_Version::supports_avx512bw());
21138 match(Set dst ( LShiftVB src shift));
21139 match(Set dst ( RShiftVB src shift));
21140 match(Set dst (URShiftVB src shift));
21141 effect(TEMP dst, TEMP vtmp);
21142 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp as TEMP" %}
21143 ins_encode %{
21144 assert(UseAVX > 2, "required");
21145
21146 int opcode = this->ideal_Opcode();
21147 int vlen_enc = vector_length_encoding(this);
21148 __ evarshiftb(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp$$XMMRegister);
21149 %}
21150 ins_pipe( pipe_slow );
21151 %}
21152
21153 instruct vshift64B_var_evex_bw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2) %{
21154 predicate(Matcher::vector_length(n) == 64 &&
21155 n->as_ShiftV()->is_var_shift() &&
21156 VM_Version::supports_avx512bw());
21157 match(Set dst ( LShiftVB src shift));
21158 match(Set dst ( RShiftVB src shift));
21159 match(Set dst (URShiftVB src shift));
21160 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
21161 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp1, $vtmp2 as TEMP" %}
21162 ins_encode %{
21163 assert(UseAVX > 2, "required");
21164
21165 int opcode = this->ideal_Opcode();
21166 int vlen_enc = Assembler::AVX_256bit;
21167 __ evarshiftb(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp1$$XMMRegister);
21168 __ vextracti64x4_high($vtmp1$$XMMRegister, $src$$XMMRegister);
21169 __ vextracti64x4_high($vtmp2$$XMMRegister, $shift$$XMMRegister);
21170 __ evarshiftb(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21171 __ vinserti64x4($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0x1);
21172 %}
21173 ins_pipe( pipe_slow );
21174 %}
21175
21176 // Short variable shift
21177 instruct vshift8S_var_nobw(vec dst, vec src, vec shift, vec vtmp) %{
21178 predicate(Matcher::vector_length(n) <= 8 &&
21179 n->as_ShiftV()->is_var_shift() &&
21180 !VM_Version::supports_avx512bw());
21181 match(Set dst ( LShiftVS src shift));
21182 match(Set dst ( RShiftVS src shift));
21183 match(Set dst (URShiftVS src shift));
21184 effect(TEMP dst, TEMP vtmp);
21185 format %{ "vector_var_shift_left_short $dst, $src, $shift\n\t" %}
21186 ins_encode %{
21187 assert(UseAVX >= 2, "required");
21188
21189 int opcode = this->ideal_Opcode();
21190 bool sign = (opcode != Op_URShiftVS);
21191 int vlen_enc = Assembler::AVX_256bit;
21192 __ vextendwd(sign, $dst$$XMMRegister, $src$$XMMRegister, 1);
21193 __ vpmovzxwd($vtmp$$XMMRegister, $shift$$XMMRegister, 1);
21194 __ varshiftd(opcode, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
21195 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21196 __ vextracti128_high($vtmp$$XMMRegister, $dst$$XMMRegister);
21197 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0);
21198 %}
21199 ins_pipe( pipe_slow );
21200 %}
21201
21202 instruct vshift16S_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2) %{
21203 predicate(Matcher::vector_length(n) == 16 &&
21204 n->as_ShiftV()->is_var_shift() &&
21205 !VM_Version::supports_avx512bw());
21206 match(Set dst ( LShiftVS src shift));
21207 match(Set dst ( RShiftVS src shift));
21208 match(Set dst (URShiftVS src shift));
21209 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
21210 format %{ "vector_var_shift_left_short $dst, $src, $shift\n\t" %}
21211 ins_encode %{
21212 assert(UseAVX >= 2, "required");
21213
21214 int opcode = this->ideal_Opcode();
21215 bool sign = (opcode != Op_URShiftVS);
21216 int vlen_enc = Assembler::AVX_256bit;
21217 // Shift lower half, with result in vtmp2 using vtmp1 as TEMP
21218 __ vextendwd(sign, $vtmp2$$XMMRegister, $src$$XMMRegister, vlen_enc);
21219 __ vpmovzxwd($vtmp1$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21220 __ varshiftd(opcode, $vtmp2$$XMMRegister, $vtmp2$$XMMRegister, $vtmp1$$XMMRegister, vlen_enc);
21221 __ vpand($vtmp2$$XMMRegister, $vtmp2$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21222
21223 // Shift upper half, with result in dst using vtmp1 as TEMP
21224 __ vextracti128_high($dst$$XMMRegister, $src$$XMMRegister);
21225 __ vextracti128_high($vtmp1$$XMMRegister, $shift$$XMMRegister);
21226 __ vextendwd(sign, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21227 __ vpmovzxwd($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, vlen_enc);
21228 __ varshiftd(opcode, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, vlen_enc);
21229 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21230
21231 // Merge lower and upper half result into dst
21232 __ vpackusdw($dst$$XMMRegister, $vtmp2$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21233 __ vpermq($dst$$XMMRegister, $dst$$XMMRegister, 0xD8, vlen_enc);
21234 %}
21235 ins_pipe( pipe_slow );
21236 %}
21237
21238 instruct vshift16S_var_evex_bw(vec dst, vec src, vec shift) %{
21239 predicate(n->as_ShiftV()->is_var_shift() &&
21240 VM_Version::supports_avx512bw());
21241 match(Set dst ( LShiftVS src shift));
21242 match(Set dst ( RShiftVS src shift));
21243 match(Set dst (URShiftVS src shift));
21244 format %{ "vector_varshift_short $dst,$src,$shift\t!" %}
21245 ins_encode %{
21246 assert(UseAVX > 2, "required");
21247
21248 int opcode = this->ideal_Opcode();
21249 int vlen_enc = vector_length_encoding(this);
21250 if (!VM_Version::supports_avx512vl()) {
21251 vlen_enc = Assembler::AVX_512bit;
21252 }
21253 __ varshiftw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21254 %}
21255 ins_pipe( pipe_slow );
21256 %}
21257
21258 //Integer variable shift
21259 instruct vshiftI_var(vec dst, vec src, vec shift) %{
21260 predicate(n->as_ShiftV()->is_var_shift());
21261 match(Set dst ( LShiftVI src shift));
21262 match(Set dst ( RShiftVI src shift));
21263 match(Set dst (URShiftVI src shift));
21264 format %{ "vector_varshift_int $dst,$src,$shift\t!" %}
21265 ins_encode %{
21266 assert(UseAVX >= 2, "required");
21267
21268 int opcode = this->ideal_Opcode();
21269 int vlen_enc = vector_length_encoding(this);
21270 __ varshiftd(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21271 %}
21272 ins_pipe( pipe_slow );
21273 %}
21274
21275 //Long variable shift
21276 instruct vshiftL_var(vec dst, vec src, vec shift) %{
21277 predicate(n->as_ShiftV()->is_var_shift());
21278 match(Set dst ( LShiftVL src shift));
21279 match(Set dst (URShiftVL src shift));
21280 format %{ "vector_varshift_long $dst,$src,$shift\t!" %}
21281 ins_encode %{
21282 assert(UseAVX >= 2, "required");
21283
21284 int opcode = this->ideal_Opcode();
21285 int vlen_enc = vector_length_encoding(this);
21286 __ varshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21287 %}
21288 ins_pipe( pipe_slow );
21289 %}
21290
21291 //Long variable right shift arithmetic
21292 instruct vshiftL_arith_var(vec dst, vec src, vec shift, vec vtmp) %{
21293 predicate(Matcher::vector_length(n) <= 4 &&
21294 n->as_ShiftV()->is_var_shift() &&
21295 UseAVX == 2);
21296 match(Set dst (RShiftVL src shift));
21297 effect(TEMP dst, TEMP vtmp);
21298 format %{ "vector_varshift_long $dst,$src,$shift\n\t! using $vtmp as TEMP" %}
21299 ins_encode %{
21300 int opcode = this->ideal_Opcode();
21301 int vlen_enc = vector_length_encoding(this);
21302 __ varshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc,
21303 $vtmp$$XMMRegister);
21304 %}
21305 ins_pipe( pipe_slow );
21306 %}
21307
21308 instruct vshiftL_arith_var_evex(vec dst, vec src, vec shift) %{
21309 predicate(n->as_ShiftV()->is_var_shift() &&
21310 UseAVX > 2);
21311 match(Set dst (RShiftVL src shift));
21312 format %{ "vector_varfshift_long $dst,$src,$shift\t!" %}
21313 ins_encode %{
21314 int opcode = this->ideal_Opcode();
21315 int vlen_enc = vector_length_encoding(this);
21316 __ varshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21317 %}
21318 ins_pipe( pipe_slow );
21319 %}
21320
21321 // --------------------------------- AND --------------------------------------
21322
21323 instruct vand(vec dst, vec src) %{
21324 predicate(UseAVX == 0);
21325 match(Set dst (AndV dst src));
21326 format %{ "pand $dst,$src\t! and vectors" %}
21327 ins_encode %{
21328 __ pand($dst$$XMMRegister, $src$$XMMRegister);
21329 %}
21330 ins_pipe( pipe_slow );
21331 %}
21332
21333 instruct vand_reg(vec dst, vec src1, vec src2) %{
21334 predicate(UseAVX > 0);
21335 match(Set dst (AndV src1 src2));
21336 format %{ "vpand $dst,$src1,$src2\t! and vectors" %}
21337 ins_encode %{
21338 int vlen_enc = vector_length_encoding(this);
21339 __ vpand($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
21340 %}
21341 ins_pipe( pipe_slow );
21342 %}
21343
21344 instruct vand_mem(vec dst, vec src, memory mem) %{
21345 predicate((UseAVX > 0) &&
21346 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
21347 match(Set dst (AndV src (LoadVector mem)));
21348 format %{ "vpand $dst,$src,$mem\t! and vectors" %}
21349 ins_encode %{
21350 int vlen_enc = vector_length_encoding(this);
21351 __ vpand($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
21352 %}
21353 ins_pipe( pipe_slow );
21354 %}
21355
21356 // --------------------------------- OR ---------------------------------------
21357
21358 instruct vor(vec dst, vec src) %{
21359 predicate(UseAVX == 0);
21360 match(Set dst (OrV dst src));
21361 format %{ "por $dst,$src\t! or vectors" %}
21362 ins_encode %{
21363 __ por($dst$$XMMRegister, $src$$XMMRegister);
21364 %}
21365 ins_pipe( pipe_slow );
21366 %}
21367
21368 instruct vor_reg(vec dst, vec src1, vec src2) %{
21369 predicate(UseAVX > 0);
21370 match(Set dst (OrV src1 src2));
21371 format %{ "vpor $dst,$src1,$src2\t! or vectors" %}
21372 ins_encode %{
21373 int vlen_enc = vector_length_encoding(this);
21374 __ vpor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
21375 %}
21376 ins_pipe( pipe_slow );
21377 %}
21378
21379 instruct vor_mem(vec dst, vec src, memory mem) %{
21380 predicate((UseAVX > 0) &&
21381 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
21382 match(Set dst (OrV src (LoadVector mem)));
21383 format %{ "vpor $dst,$src,$mem\t! or vectors" %}
21384 ins_encode %{
21385 int vlen_enc = vector_length_encoding(this);
21386 __ vpor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
21387 %}
21388 ins_pipe( pipe_slow );
21389 %}
21390
21391 // --------------------------------- XOR --------------------------------------
21392
21393 instruct vxor(vec dst, vec src) %{
21394 predicate(UseAVX == 0);
21395 match(Set dst (XorV dst src));
21396 format %{ "pxor $dst,$src\t! xor vectors" %}
21397 ins_encode %{
21398 __ pxor($dst$$XMMRegister, $src$$XMMRegister);
21399 %}
21400 ins_pipe( pipe_slow );
21401 %}
21402
21403 instruct vxor_reg(vec dst, vec src1, vec src2) %{
21404 predicate(UseAVX > 0);
21405 match(Set dst (XorV src1 src2));
21406 format %{ "vpxor $dst,$src1,$src2\t! xor vectors" %}
21407 ins_encode %{
21408 int vlen_enc = vector_length_encoding(this);
21409 __ vpxor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
21410 %}
21411 ins_pipe( pipe_slow );
21412 %}
21413
21414 instruct vxor_mem(vec dst, vec src, memory mem) %{
21415 predicate((UseAVX > 0) &&
21416 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
21417 match(Set dst (XorV src (LoadVector mem)));
21418 format %{ "vpxor $dst,$src,$mem\t! xor vectors" %}
21419 ins_encode %{
21420 int vlen_enc = vector_length_encoding(this);
21421 __ vpxor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
21422 %}
21423 ins_pipe( pipe_slow );
21424 %}
21425
21426 // --------------------------------- VectorCast --------------------------------------
21427
21428 instruct vcastBtoX(vec dst, vec src) %{
21429 predicate(VM_Version::supports_avx512vl() || Matcher::vector_element_basic_type(n) != T_DOUBLE);
21430 match(Set dst (VectorCastB2X src));
21431 format %{ "vector_cast_b2x $dst,$src\t!" %}
21432 ins_encode %{
21433 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21434 int vlen_enc = vector_length_encoding(this);
21435 __ vconvert_b2x(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21436 %}
21437 ins_pipe( pipe_slow );
21438 %}
21439
21440 instruct vcastBtoD(legVec dst, legVec src) %{
21441 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_element_basic_type(n) == T_DOUBLE);
21442 match(Set dst (VectorCastB2X src));
21443 format %{ "vector_cast_b2x $dst,$src\t!" %}
21444 ins_encode %{
21445 int vlen_enc = vector_length_encoding(this);
21446 __ vconvert_b2x(T_DOUBLE, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21447 %}
21448 ins_pipe( pipe_slow );
21449 %}
21450
21451 instruct castStoX(vec dst, vec src) %{
21452 predicate((UseAVX <= 2 || !VM_Version::supports_avx512vlbw()) &&
21453 Matcher::vector_length(n->in(1)) <= 8 && // src
21454 Matcher::vector_element_basic_type(n) == T_BYTE);
21455 match(Set dst (VectorCastS2X src));
21456 format %{ "vector_cast_s2x $dst,$src" %}
21457 ins_encode %{
21458 assert(UseAVX > 0, "required");
21459
21460 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), 0, noreg);
21461 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, 0);
21462 %}
21463 ins_pipe( pipe_slow );
21464 %}
21465
21466 instruct vcastStoX(vec dst, vec src, vec vtmp) %{
21467 predicate((UseAVX <= 2 || !VM_Version::supports_avx512vlbw()) &&
21468 Matcher::vector_length(n->in(1)) == 16 && // src
21469 Matcher::vector_element_basic_type(n) == T_BYTE);
21470 effect(TEMP dst, TEMP vtmp);
21471 match(Set dst (VectorCastS2X src));
21472 format %{ "vector_cast_s2x $dst,$src\t! using $vtmp as TEMP" %}
21473 ins_encode %{
21474 assert(UseAVX > 0, "required");
21475
21476 int vlen_enc = vector_length_encoding(Matcher::vector_length_in_bytes(this, $src));
21477 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
21478 __ vextracti128($vtmp$$XMMRegister, $dst$$XMMRegister, 0x1);
21479 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0);
21480 %}
21481 ins_pipe( pipe_slow );
21482 %}
21483
21484 instruct vcastStoX_evex(vec dst, vec src) %{
21485 predicate((UseAVX > 2 && VM_Version::supports_avx512vlbw()) ||
21486 (Matcher::vector_length_in_bytes(n) >= Matcher::vector_length_in_bytes(n->in(1)))); // dst >= src
21487 match(Set dst (VectorCastS2X src));
21488 format %{ "vector_cast_s2x $dst,$src\t!" %}
21489 ins_encode %{
21490 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21491 int src_vlen_enc = vector_length_encoding(this, $src);
21492 int vlen_enc = vector_length_encoding(this);
21493 switch (to_elem_bt) {
21494 case T_BYTE:
21495 if (!VM_Version::supports_avx512vl()) {
21496 vlen_enc = Assembler::AVX_512bit;
21497 }
21498 __ evpmovwb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
21499 break;
21500 case T_INT:
21501 __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21502 break;
21503 case T_FLOAT:
21504 __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21505 __ vcvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21506 break;
21507 case T_LONG:
21508 __ vpmovsxwq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21509 break;
21510 case T_DOUBLE: {
21511 int mid_vlen_enc = (vlen_enc == Assembler::AVX_512bit) ? Assembler::AVX_256bit : Assembler::AVX_128bit;
21512 __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, mid_vlen_enc);
21513 __ vcvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21514 break;
21515 }
21516 default:
21517 ShouldNotReachHere();
21518 }
21519 %}
21520 ins_pipe( pipe_slow );
21521 %}
21522
21523 instruct castItoX(vec dst, vec src) %{
21524 predicate(UseAVX <= 2 &&
21525 (Matcher::vector_length_in_bytes(n->in(1)) <= 16) &&
21526 (Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)))); // dst < src
21527 match(Set dst (VectorCastI2X src));
21528 format %{ "vector_cast_i2x $dst,$src" %}
21529 ins_encode %{
21530 assert(UseAVX > 0, "required");
21531
21532 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21533 int vlen_enc = vector_length_encoding(this, $src);
21534
21535 if (to_elem_bt == T_BYTE) {
21536 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_byte_mask()), vlen_enc, noreg);
21537 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21538 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21539 } else {
21540 assert(to_elem_bt == T_SHORT, "%s", type2name(to_elem_bt));
21541 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21542 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21543 }
21544 %}
21545 ins_pipe( pipe_slow );
21546 %}
21547
21548 instruct vcastItoX(vec dst, vec src, vec vtmp) %{
21549 predicate(UseAVX <= 2 &&
21550 (Matcher::vector_length_in_bytes(n->in(1)) == 32) &&
21551 (Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)))); // dst < src
21552 match(Set dst (VectorCastI2X src));
21553 format %{ "vector_cast_i2x $dst,$src\t! using $vtmp as TEMP" %}
21554 effect(TEMP dst, TEMP vtmp);
21555 ins_encode %{
21556 assert(UseAVX > 0, "required");
21557
21558 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21559 int vlen_enc = vector_length_encoding(this, $src);
21560
21561 if (to_elem_bt == T_BYTE) {
21562 __ vpand($vtmp$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_byte_mask()), vlen_enc, noreg);
21563 __ vextracti128($dst$$XMMRegister, $vtmp$$XMMRegister, 0x1);
21564 __ vpackusdw($dst$$XMMRegister, $vtmp$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21565 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21566 } else {
21567 assert(to_elem_bt == T_SHORT, "%s", type2name(to_elem_bt));
21568 __ vpand($vtmp$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21569 __ vextracti128($dst$$XMMRegister, $vtmp$$XMMRegister, 0x1);
21570 __ vpackusdw($dst$$XMMRegister, $vtmp$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21571 }
21572 %}
21573 ins_pipe( pipe_slow );
21574 %}
21575
21576 instruct vcastItoX_evex(vec dst, vec src) %{
21577 predicate(UseAVX > 2 ||
21578 (Matcher::vector_length_in_bytes(n) >= Matcher::vector_length_in_bytes(n->in(1)))); // dst >= src
21579 match(Set dst (VectorCastI2X src));
21580 format %{ "vector_cast_i2x $dst,$src\t!" %}
21581 ins_encode %{
21582 assert(UseAVX > 0, "required");
21583
21584 BasicType dst_elem_bt = Matcher::vector_element_basic_type(this);
21585 int src_vlen_enc = vector_length_encoding(this, $src);
21586 int dst_vlen_enc = vector_length_encoding(this);
21587 switch (dst_elem_bt) {
21588 case T_BYTE:
21589 if (!VM_Version::supports_avx512vl()) {
21590 src_vlen_enc = Assembler::AVX_512bit;
21591 }
21592 __ evpmovdb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
21593 break;
21594 case T_SHORT:
21595 if (!VM_Version::supports_avx512vl()) {
21596 src_vlen_enc = Assembler::AVX_512bit;
21597 }
21598 __ evpmovdw($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
21599 break;
21600 case T_FLOAT:
21601 __ vcvtdq2ps($dst$$XMMRegister, $src$$XMMRegister, dst_vlen_enc);
21602 break;
21603 case T_LONG:
21604 __ vpmovsxdq($dst$$XMMRegister, $src$$XMMRegister, dst_vlen_enc);
21605 break;
21606 case T_DOUBLE:
21607 __ vcvtdq2pd($dst$$XMMRegister, $src$$XMMRegister, dst_vlen_enc);
21608 break;
21609 default:
21610 ShouldNotReachHere();
21611 }
21612 %}
21613 ins_pipe( pipe_slow );
21614 %}
21615
21616 instruct vcastLtoBS(vec dst, vec src) %{
21617 predicate((Matcher::vector_element_basic_type(n) == T_BYTE || Matcher::vector_element_basic_type(n) == T_SHORT) &&
21618 UseAVX <= 2);
21619 match(Set dst (VectorCastL2X src));
21620 format %{ "vector_cast_l2x $dst,$src" %}
21621 ins_encode %{
21622 assert(UseAVX > 0, "required");
21623
21624 int vlen = Matcher::vector_length_in_bytes(this, $src);
21625 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21626 AddressLiteral mask_addr = (to_elem_bt == T_BYTE) ? ExternalAddress(vector_int_to_byte_mask())
21627 : ExternalAddress(vector_int_to_short_mask());
21628 if (vlen <= 16) {
21629 __ vpshufd($dst$$XMMRegister, $src$$XMMRegister, 8, Assembler::AVX_128bit);
21630 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, mask_addr, Assembler::AVX_128bit, noreg);
21631 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21632 } else {
21633 assert(vlen <= 32, "required");
21634 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 8, Assembler::AVX_256bit);
21635 __ vpermpd($dst$$XMMRegister, $dst$$XMMRegister, 8, Assembler::AVX_256bit);
21636 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, mask_addr, Assembler::AVX_128bit, noreg);
21637 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21638 }
21639 if (to_elem_bt == T_BYTE) {
21640 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21641 }
21642 %}
21643 ins_pipe( pipe_slow );
21644 %}
21645
21646 instruct vcastLtoX_evex(vec dst, vec src) %{
21647 predicate(UseAVX > 2 ||
21648 (Matcher::vector_element_basic_type(n) == T_INT ||
21649 Matcher::vector_element_basic_type(n) == T_FLOAT ||
21650 Matcher::vector_element_basic_type(n) == T_DOUBLE));
21651 match(Set dst (VectorCastL2X src));
21652 format %{ "vector_cast_l2x $dst,$src\t!" %}
21653 ins_encode %{
21654 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21655 int vlen = Matcher::vector_length_in_bytes(this, $src);
21656 int vlen_enc = vector_length_encoding(this, $src);
21657 switch (to_elem_bt) {
21658 case T_BYTE:
21659 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
21660 vlen_enc = Assembler::AVX_512bit;
21661 }
21662 __ evpmovqb($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21663 break;
21664 case T_SHORT:
21665 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
21666 vlen_enc = Assembler::AVX_512bit;
21667 }
21668 __ evpmovqw($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21669 break;
21670 case T_INT:
21671 if (vlen == 8) {
21672 if ($dst$$XMMRegister != $src$$XMMRegister) {
21673 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
21674 }
21675 } else if (vlen == 16) {
21676 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 8);
21677 } else if (vlen == 32) {
21678 if (UseAVX > 2) {
21679 if (!VM_Version::supports_avx512vl()) {
21680 vlen_enc = Assembler::AVX_512bit;
21681 }
21682 __ evpmovqd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21683 } else {
21684 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 8, vlen_enc);
21685 __ vpermpd($dst$$XMMRegister, $dst$$XMMRegister, 8, vlen_enc);
21686 }
21687 } else { // vlen == 64
21688 __ evpmovqd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21689 }
21690 break;
21691 case T_FLOAT:
21692 assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "required");
21693 __ evcvtqq2ps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21694 break;
21695 case T_DOUBLE:
21696 assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "required");
21697 __ evcvtqq2pd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21698 break;
21699
21700 default: assert(false, "%s", type2name(to_elem_bt));
21701 }
21702 %}
21703 ins_pipe( pipe_slow );
21704 %}
21705
21706 instruct vcastFtoD_reg(vec dst, vec src) %{
21707 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
21708 match(Set dst (VectorCastF2X src));
21709 format %{ "vector_cast_f2d $dst,$src\t!" %}
21710 ins_encode %{
21711 int vlen_enc = vector_length_encoding(this);
21712 __ vcvtps2pd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21713 %}
21714 ins_pipe( pipe_slow );
21715 %}
21716
21717
21718 instruct castFtoX_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, rFlagsReg cr) %{
21719 predicate(!VM_Version::supports_avx10_2() &&
21720 !VM_Version::supports_avx512vl() &&
21721 Matcher::vector_length_in_bytes(n->in(1)) < 64 &&
21722 type2aelembytes(Matcher::vector_element_basic_type(n)) <= 4 &&
21723 is_integral_type(Matcher::vector_element_basic_type(n)));
21724 match(Set dst (VectorCastF2X src));
21725 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, KILL cr);
21726 format %{ "vector_cast_f2x $dst,$src\t! using $xtmp1, $xtmp2, $xtmp3 and $xtmp4 as TEMP" %}
21727 ins_encode %{
21728 int vlen_enc = vector_length_encoding(this, $src);
21729 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21730 // JDK-8292878 removed the need for an explicit scratch register needed to load greater than
21731 // 32 bit addresses for register indirect addressing mode since stub constants
21732 // are part of code cache and there is a cap of 2G on ReservedCodeCacheSize currently.
21733 // However, targets are free to increase this limit, but having a large code cache size
21734 // greater than 2G looks unreasonable in practical scenario, on the hind side with given
21735 // cap we save a temporary register allocation which in limiting case can prevent
21736 // spilling in high register pressure blocks.
21737 __ vector_castF2X_avx(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
21738 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister,
21739 ExternalAddress(vector_float_signflip()), noreg, vlen_enc);
21740 %}
21741 ins_pipe( pipe_slow );
21742 %}
21743
21744 instruct castFtoX_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
21745 predicate(!VM_Version::supports_avx10_2() &&
21746 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n->in(1)) == 64) &&
21747 is_integral_type(Matcher::vector_element_basic_type(n)));
21748 match(Set dst (VectorCastF2X src));
21749 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
21750 format %{ "vector_cast_f2x $dst,$src\t! using $xtmp1, $xtmp2, $ktmp1 and $ktmp2 as TEMP" %}
21751 ins_encode %{
21752 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21753 if (to_elem_bt == T_LONG) {
21754 int vlen_enc = vector_length_encoding(this);
21755 __ vector_castF2L_evex($dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
21756 $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister,
21757 ExternalAddress(vector_double_signflip()), noreg, vlen_enc);
21758 } else {
21759 int vlen_enc = vector_length_encoding(this, $src);
21760 __ vector_castF2X_evex(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
21761 $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister,
21762 ExternalAddress(vector_float_signflip()), noreg, vlen_enc);
21763 }
21764 %}
21765 ins_pipe( pipe_slow );
21766 %}
21767
21768 instruct castFtoX_reg_avx10_2(vec dst, vec src) %{
21769 predicate(VM_Version::supports_avx10_2() &&
21770 is_integral_type(Matcher::vector_element_basic_type(n)));
21771 match(Set dst (VectorCastF2X src));
21772 format %{ "vector_cast_f2x_avx10_2 $dst, $src\t!" %}
21773 ins_encode %{
21774 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21775 int vlen_enc = (to_elem_bt == T_LONG) ? vector_length_encoding(this) : vector_length_encoding(this, $src);
21776 __ vector_castF2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21777 %}
21778 ins_pipe( pipe_slow );
21779 %}
21780
21781 instruct castFtoX_mem_avx10_2(vec dst, memory src) %{
21782 predicate(VM_Version::supports_avx10_2() &&
21783 is_integral_type(Matcher::vector_element_basic_type(n)));
21784 match(Set dst (VectorCastF2X (LoadVector src)));
21785 format %{ "vector_cast_f2x_avx10_2 $dst, $src\t!" %}
21786 ins_encode %{
21787 int vlen = Matcher::vector_length(this);
21788 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21789 int vlen_enc = (to_elem_bt == T_LONG) ? vector_length_encoding(this) : vector_length_encoding(vlen * sizeof(jfloat));
21790 __ vector_castF2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$Address, vlen_enc);
21791 %}
21792 ins_pipe( pipe_slow );
21793 %}
21794
21795 instruct vcastDtoF_reg(vec dst, vec src) %{
21796 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
21797 match(Set dst (VectorCastD2X src));
21798 format %{ "vector_cast_d2x $dst,$src\t!" %}
21799 ins_encode %{
21800 int vlen_enc = vector_length_encoding(this, $src);
21801 __ vcvtpd2ps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21802 %}
21803 ins_pipe( pipe_slow );
21804 %}
21805
21806 instruct castDtoX_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, vec xtmp5, rFlagsReg cr) %{
21807 predicate(!VM_Version::supports_avx10_2() &&
21808 !VM_Version::supports_avx512vl() &&
21809 Matcher::vector_length_in_bytes(n->in(1)) < 64 &&
21810 is_integral_type(Matcher::vector_element_basic_type(n)));
21811 match(Set dst (VectorCastD2X src));
21812 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, TEMP xtmp5, KILL cr);
21813 format %{ "vector_cast_d2x $dst,$src\t! using $xtmp1, $xtmp2, $xtmp3, $xtmp4 and $xtmp5 as TEMP" %}
21814 ins_encode %{
21815 int vlen_enc = vector_length_encoding(this, $src);
21816 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21817 __ vector_castD2X_avx(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
21818 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister, $xtmp5$$XMMRegister,
21819 ExternalAddress(vector_float_signflip()), noreg, vlen_enc);
21820 %}
21821 ins_pipe( pipe_slow );
21822 %}
21823
21824 instruct castDtoX_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
21825 predicate(!VM_Version::supports_avx10_2() &&
21826 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n->in(1)) == 64) &&
21827 is_integral_type(Matcher::vector_element_basic_type(n)));
21828 match(Set dst (VectorCastD2X src));
21829 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
21830 format %{ "vector_cast_d2x $dst,$src\t! using $xtmp1, $xtmp2, $ktmp1 and $ktmp2 as TEMP" %}
21831 ins_encode %{
21832 int vlen_enc = vector_length_encoding(this, $src);
21833 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21834 AddressLiteral signflip = VM_Version::supports_avx512dq() ? ExternalAddress(vector_double_signflip()) :
21835 ExternalAddress(vector_float_signflip());
21836 __ vector_castD2X_evex(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
21837 $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister, signflip, noreg, vlen_enc);
21838 %}
21839 ins_pipe( pipe_slow );
21840 %}
21841
21842 instruct castDtoX_reg_avx10_2(vec dst, vec src) %{
21843 predicate(VM_Version::supports_avx10_2() &&
21844 is_integral_type(Matcher::vector_element_basic_type(n)));
21845 match(Set dst (VectorCastD2X src));
21846 format %{ "vector_cast_d2x_avx10_2 $dst, $src\t!" %}
21847 ins_encode %{
21848 int vlen_enc = vector_length_encoding(this, $src);
21849 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21850 __ vector_castD2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21851 %}
21852 ins_pipe( pipe_slow );
21853 %}
21854
21855 instruct castDtoX_mem_avx10_2(vec dst, memory src) %{
21856 predicate(VM_Version::supports_avx10_2() &&
21857 is_integral_type(Matcher::vector_element_basic_type(n)));
21858 match(Set dst (VectorCastD2X (LoadVector src)));
21859 format %{ "vector_cast_d2x_avx10_2 $dst, $src\t!" %}
21860 ins_encode %{
21861 int vlen = Matcher::vector_length(this);
21862 int vlen_enc = vector_length_encoding(vlen * sizeof(jdouble));
21863 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21864 __ vector_castD2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$Address, vlen_enc);
21865 %}
21866 ins_pipe( pipe_slow );
21867 %}
21868
21869 instruct vucast(vec dst, vec src) %{
21870 match(Set dst (VectorUCastB2X src));
21871 match(Set dst (VectorUCastS2X src));
21872 match(Set dst (VectorUCastI2X src));
21873 format %{ "vector_ucast $dst,$src\t!" %}
21874 ins_encode %{
21875 assert(UseAVX > 0, "required");
21876
21877 BasicType from_elem_bt = Matcher::vector_element_basic_type(this, $src);
21878 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21879 int vlen_enc = vector_length_encoding(this);
21880 __ vector_unsigned_cast($dst$$XMMRegister, $src$$XMMRegister, vlen_enc, from_elem_bt, to_elem_bt);
21881 %}
21882 ins_pipe( pipe_slow );
21883 %}
21884
21885 instruct vround_float_avx(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, rFlagsReg cr) %{
21886 predicate(!VM_Version::supports_avx512vl() &&
21887 Matcher::vector_length_in_bytes(n) < 64 &&
21888 Matcher::vector_element_basic_type(n) == T_INT);
21889 match(Set dst (RoundVF src));
21890 effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, KILL cr);
21891 format %{ "vector_round_float $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $xtmp3, $xtmp4 as TEMP" %}
21892 ins_encode %{
21893 int vlen_enc = vector_length_encoding(this);
21894 InternalAddress new_mxcsr = $constantaddress((jint)(EnableX86ECoreOpts ? 0x3FBF : 0x3F80));
21895 __ vector_round_float_avx($dst$$XMMRegister, $src$$XMMRegister,
21896 ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), new_mxcsr, vlen_enc,
21897 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister);
21898 %}
21899 ins_pipe( pipe_slow );
21900 %}
21901
21902 instruct vround_float_evex(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
21903 predicate((VM_Version::supports_avx512vl() ||
21904 Matcher::vector_length_in_bytes(n) == 64) &&
21905 Matcher::vector_element_basic_type(n) == T_INT);
21906 match(Set dst (RoundVF src));
21907 effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
21908 format %{ "vector_round_float $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $ktmp1, $ktmp2 as TEMP" %}
21909 ins_encode %{
21910 int vlen_enc = vector_length_encoding(this);
21911 InternalAddress new_mxcsr = $constantaddress((jint)(EnableX86ECoreOpts ? 0x3FBF : 0x3F80));
21912 __ vector_round_float_evex($dst$$XMMRegister, $src$$XMMRegister,
21913 ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), new_mxcsr, vlen_enc,
21914 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
21915 %}
21916 ins_pipe( pipe_slow );
21917 %}
21918
21919 instruct vround_reg_evex(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
21920 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
21921 match(Set dst (RoundVD src));
21922 effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
21923 format %{ "vector_round_long $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $ktmp1, $ktmp2 as TEMP" %}
21924 ins_encode %{
21925 int vlen_enc = vector_length_encoding(this);
21926 InternalAddress new_mxcsr = $constantaddress((jint)(EnableX86ECoreOpts ? 0x3FBF : 0x3F80));
21927 __ vector_round_double_evex($dst$$XMMRegister, $src$$XMMRegister,
21928 ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), new_mxcsr, vlen_enc,
21929 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
21930 %}
21931 ins_pipe( pipe_slow );
21932 %}
21933
21934 // --------------------------------- VectorMaskCmp --------------------------------------
21935
21936 instruct vcmpFD(legVec dst, legVec src1, legVec src2, immI8 cond) %{
21937 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
21938 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 8 && // src1
21939 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
21940 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
21941 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
21942 format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
21943 ins_encode %{
21944 int vlen_enc = vector_length_encoding(this, $src1);
21945 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
21946 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
21947 __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
21948 } else {
21949 __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
21950 }
21951 %}
21952 ins_pipe( pipe_slow );
21953 %}
21954
21955 instruct evcmpFD64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
21956 predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64 && // src1
21957 n->bottom_type()->isa_pvectmask() == nullptr &&
21958 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
21959 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
21960 effect(TEMP ktmp);
21961 format %{ "vector_compare $dst,$src1,$src2,$cond" %}
21962 ins_encode %{
21963 int vlen_enc = Assembler::AVX_512bit;
21964 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
21965 KRegister mask = k0; // The comparison itself is not being masked.
21966 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
21967 __ evcmpps($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
21968 __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, noreg);
21969 } else {
21970 __ evcmppd($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
21971 __ evmovdquq($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, noreg);
21972 }
21973 %}
21974 ins_pipe( pipe_slow );
21975 %}
21976
21977 instruct evcmpFD(kReg dst, vec src1, vec src2, immI8 cond) %{
21978 predicate(n->bottom_type()->isa_pvectmask() &&
21979 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
21980 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
21981 format %{ "vector_compare_evex $dst,$src1,$src2,$cond\t!" %}
21982 ins_encode %{
21983 assert(bottom_type()->isa_pvectmask(), "TypePVectMask expected");
21984 int vlen_enc = vector_length_encoding(this, $src1);
21985 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
21986 KRegister mask = k0; // The comparison itself is not being masked.
21987 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
21988 __ evcmpps($dst$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
21989 } else {
21990 __ evcmppd($dst$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
21991 }
21992 %}
21993 ins_pipe( pipe_slow );
21994 %}
21995
21996 instruct vcmp_direct(legVec dst, legVec src1, legVec src2, immI8 cond) %{
21997 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
21998 !Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
21999 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
22000 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
22001 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1))) &&
22002 (n->in(2)->get_int() == BoolTest::eq ||
22003 n->in(2)->get_int() == BoolTest::lt ||
22004 n->in(2)->get_int() == BoolTest::gt)); // cond
22005 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22006 format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
22007 ins_encode %{
22008 int vlen_enc = vector_length_encoding(this, $src1);
22009 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22010 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
22011 __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, cmp, ww, vlen_enc);
22012 %}
22013 ins_pipe( pipe_slow );
22014 %}
22015
22016 instruct vcmp_negate(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
22017 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
22018 !Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
22019 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
22020 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
22021 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1))) &&
22022 (n->in(2)->get_int() == BoolTest::ne ||
22023 n->in(2)->get_int() == BoolTest::le ||
22024 n->in(2)->get_int() == BoolTest::ge)); // cond
22025 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22026 effect(TEMP dst, TEMP xtmp);
22027 format %{ "vector_compare $dst,$src1,$src2,$cond\t! using $xtmp as TEMP" %}
22028 ins_encode %{
22029 int vlen_enc = vector_length_encoding(this, $src1);
22030 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22031 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
22032 __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $xtmp$$XMMRegister, cmp, ww, vlen_enc);
22033 %}
22034 ins_pipe( pipe_slow );
22035 %}
22036
22037 instruct vcmpu(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
22038 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
22039 Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
22040 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
22041 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
22042 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
22043 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22044 effect(TEMP dst, TEMP xtmp);
22045 format %{ "vector_compareu $dst,$src1,$src2,$cond\t! using $xtmp as TEMP" %}
22046 ins_encode %{
22047 InternalAddress flip_bit = $constantaddress(high_bit_set(Matcher::vector_element_basic_type(this, $src1)));
22048 int vlen_enc = vector_length_encoding(this, $src1);
22049 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22050 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
22051
22052 if (vlen_enc == Assembler::AVX_128bit) {
22053 __ vmovddup($xtmp$$XMMRegister, flip_bit, vlen_enc, noreg);
22054 } else {
22055 __ vbroadcastsd($xtmp$$XMMRegister, flip_bit, vlen_enc, noreg);
22056 }
22057 __ vpxor($dst$$XMMRegister, $xtmp$$XMMRegister, $src1$$XMMRegister, vlen_enc);
22058 __ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $src2$$XMMRegister, vlen_enc);
22059 __ vpcmpCCW($dst$$XMMRegister, $dst$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, cmp, ww, vlen_enc);
22060 %}
22061 ins_pipe( pipe_slow );
22062 %}
22063
22064 instruct vcmp64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
22065 predicate((n->bottom_type()->isa_pvectmask() == nullptr &&
22066 Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64) && // src1
22067 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
22068 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22069 effect(TEMP ktmp);
22070 format %{ "vector_compare $dst,$src1,$src2,$cond" %}
22071 ins_encode %{
22072 assert(UseAVX > 2, "required");
22073
22074 int vlen_enc = vector_length_encoding(this, $src1);
22075 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22076 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
22077 KRegister mask = k0; // The comparison itself is not being masked.
22078 bool merge = false;
22079 BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
22080
22081 switch (src1_elem_bt) {
22082 case T_INT: {
22083 __ evpcmpd($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22084 __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), merge, vlen_enc, noreg);
22085 break;
22086 }
22087 case T_LONG: {
22088 __ evpcmpq($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22089 __ evmovdquq($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), merge, vlen_enc, noreg);
22090 break;
22091 }
22092 default: assert(false, "%s", type2name(src1_elem_bt));
22093 }
22094 %}
22095 ins_pipe( pipe_slow );
22096 %}
22097
22098
22099 instruct evcmp(kReg dst, vec src1, vec src2, immI8 cond) %{
22100 predicate(n->bottom_type()->isa_pvectmask() &&
22101 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
22102 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22103 format %{ "vector_compared_evex $dst,$src1,$src2,$cond\t!" %}
22104 ins_encode %{
22105 assert(UseAVX > 2, "required");
22106 assert(bottom_type()->isa_pvectmask(), "TypePVectMask expected");
22107
22108 int vlen_enc = vector_length_encoding(this, $src1);
22109 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22110 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
22111 BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
22112
22113 // Comparison i
22114 switch (src1_elem_bt) {
22115 case T_BYTE: {
22116 __ evpcmpb($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22117 break;
22118 }
22119 case T_SHORT: {
22120 __ evpcmpw($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22121 break;
22122 }
22123 case T_INT: {
22124 __ evpcmpd($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22125 break;
22126 }
22127 case T_LONG: {
22128 __ evpcmpq($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22129 break;
22130 }
22131 default: assert(false, "%s", type2name(src1_elem_bt));
22132 }
22133 %}
22134 ins_pipe( pipe_slow );
22135 %}
22136
22137 // Extract
22138
22139 instruct extractI(rRegI dst, legVec src, immU8 idx) %{
22140 predicate(Matcher::vector_length_in_bytes(n->in(1)) <= 16); // src
22141 match(Set dst (ExtractI src idx));
22142 match(Set dst (ExtractS src idx));
22143 match(Set dst (ExtractB src idx));
22144 format %{ "extractI $dst,$src,$idx\t!" %}
22145 ins_encode %{
22146 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22147
22148 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
22149 __ get_elem(elem_bt, $dst$$Register, $src$$XMMRegister, $idx$$constant);
22150 %}
22151 ins_pipe( pipe_slow );
22152 %}
22153
22154 instruct vextractI(rRegI dst, legVec src, immI idx, legVec vtmp) %{
22155 predicate(Matcher::vector_length_in_bytes(n->in(1)) == 32 || // src
22156 Matcher::vector_length_in_bytes(n->in(1)) == 64); // src
22157 match(Set dst (ExtractI src idx));
22158 match(Set dst (ExtractS src idx));
22159 match(Set dst (ExtractB src idx));
22160 effect(TEMP vtmp);
22161 format %{ "vextractI $dst,$src,$idx\t! using $vtmp as TEMP" %}
22162 ins_encode %{
22163 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22164
22165 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
22166 XMMRegister lane_xmm = __ get_lane(elem_bt, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22167 __ get_elem(elem_bt, $dst$$Register, lane_xmm, $idx$$constant);
22168 %}
22169 ins_pipe( pipe_slow );
22170 %}
22171
22172 instruct extractL(rRegL dst, legVec src, immU8 idx) %{
22173 predicate(Matcher::vector_length(n->in(1)) <= 2); // src
22174 match(Set dst (ExtractL src idx));
22175 format %{ "extractL $dst,$src,$idx\t!" %}
22176 ins_encode %{
22177 assert(UseSSE >= 4, "required");
22178 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22179
22180 __ get_elem(T_LONG, $dst$$Register, $src$$XMMRegister, $idx$$constant);
22181 %}
22182 ins_pipe( pipe_slow );
22183 %}
22184
22185 instruct vextractL(rRegL dst, legVec src, immU8 idx, legVec vtmp) %{
22186 predicate(Matcher::vector_length(n->in(1)) == 4 || // src
22187 Matcher::vector_length(n->in(1)) == 8); // src
22188 match(Set dst (ExtractL src idx));
22189 effect(TEMP vtmp);
22190 format %{ "vextractL $dst,$src,$idx\t! using $vtmp as TEMP" %}
22191 ins_encode %{
22192 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22193
22194 XMMRegister lane_reg = __ get_lane(T_LONG, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22195 __ get_elem(T_LONG, $dst$$Register, lane_reg, $idx$$constant);
22196 %}
22197 ins_pipe( pipe_slow );
22198 %}
22199
22200 instruct extractF(legRegF dst, legVec src, immU8 idx, legVec vtmp) %{
22201 predicate(Matcher::vector_length(n->in(1)) <= 4);
22202 match(Set dst (ExtractF src idx));
22203 effect(TEMP dst, TEMP vtmp);
22204 format %{ "extractF $dst,$src,$idx\t! using $vtmp as TEMP" %}
22205 ins_encode %{
22206 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22207
22208 __ get_elem(T_FLOAT, $dst$$XMMRegister, $src$$XMMRegister, $idx$$constant, $vtmp$$XMMRegister);
22209 %}
22210 ins_pipe( pipe_slow );
22211 %}
22212
22213 instruct vextractF(legRegF dst, legVec src, immU8 idx, legVec vtmp) %{
22214 predicate(Matcher::vector_length(n->in(1)/*src*/) == 8 ||
22215 Matcher::vector_length(n->in(1)/*src*/) == 16);
22216 match(Set dst (ExtractF src idx));
22217 effect(TEMP vtmp);
22218 format %{ "vextractF $dst,$src,$idx\t! using $vtmp as TEMP" %}
22219 ins_encode %{
22220 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22221
22222 XMMRegister lane_reg = __ get_lane(T_FLOAT, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22223 __ get_elem(T_FLOAT, $dst$$XMMRegister, lane_reg, $idx$$constant);
22224 %}
22225 ins_pipe( pipe_slow );
22226 %}
22227
22228 instruct extractD(legRegD dst, legVec src, immU8 idx) %{
22229 predicate(Matcher::vector_length(n->in(1)) == 2); // src
22230 match(Set dst (ExtractD src idx));
22231 format %{ "extractD $dst,$src,$idx\t!" %}
22232 ins_encode %{
22233 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22234
22235 __ get_elem(T_DOUBLE, $dst$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22236 %}
22237 ins_pipe( pipe_slow );
22238 %}
22239
22240 instruct vextractD(legRegD dst, legVec src, immU8 idx, legVec vtmp) %{
22241 predicate(Matcher::vector_length(n->in(1)) == 4 || // src
22242 Matcher::vector_length(n->in(1)) == 8); // src
22243 match(Set dst (ExtractD src idx));
22244 effect(TEMP vtmp);
22245 format %{ "vextractD $dst,$src,$idx\t! using $vtmp as TEMP" %}
22246 ins_encode %{
22247 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22248
22249 XMMRegister lane_reg = __ get_lane(T_DOUBLE, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22250 __ get_elem(T_DOUBLE, $dst$$XMMRegister, lane_reg, $idx$$constant);
22251 %}
22252 ins_pipe( pipe_slow );
22253 %}
22254
22255 // --------------------------------- Vector Blend --------------------------------------
22256
22257 instruct blendvp(vec dst, vec src, vec mask, rxmm0 tmp) %{
22258 predicate(UseAVX == 0);
22259 match(Set dst (VectorBlend (Binary dst src) mask));
22260 format %{ "vector_blend $dst,$src,$mask\t! using $tmp as TEMP" %}
22261 effect(TEMP tmp);
22262 ins_encode %{
22263 assert(UseSSE >= 4, "required");
22264
22265 if ($mask$$XMMRegister != $tmp$$XMMRegister) {
22266 __ movdqu($tmp$$XMMRegister, $mask$$XMMRegister);
22267 }
22268 __ pblendvb($dst$$XMMRegister, $src$$XMMRegister); // uses xmm0 as mask
22269 %}
22270 ins_pipe( pipe_slow );
22271 %}
22272
22273 instruct vblendvpI(legVec dst, legVec src1, legVec src2, legVec mask) %{
22274 predicate(UseAVX > 0 && !EnableX86ECoreOpts &&
22275 n->in(2)->bottom_type()->isa_pvectmask() == nullptr &&
22276 Matcher::vector_length_in_bytes(n) <= 32 &&
22277 is_integral_type(Matcher::vector_element_basic_type(n)));
22278 match(Set dst (VectorBlend (Binary src1 src2) mask));
22279 format %{ "vector_blend $dst,$src1,$src2,$mask\t!" %}
22280 ins_encode %{
22281 int vlen_enc = vector_length_encoding(this);
22282 __ vpblendvb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $mask$$XMMRegister, vlen_enc);
22283 %}
22284 ins_pipe( pipe_slow );
22285 %}
22286
22287 instruct vblendvpFD(legVec dst, legVec src1, legVec src2, legVec mask) %{
22288 predicate(UseAVX > 0 && !EnableX86ECoreOpts &&
22289 n->in(2)->bottom_type()->isa_pvectmask() == nullptr &&
22290 Matcher::vector_length_in_bytes(n) <= 32 &&
22291 !is_integral_type(Matcher::vector_element_basic_type(n)));
22292 match(Set dst (VectorBlend (Binary src1 src2) mask));
22293 format %{ "vector_blend $dst,$src1,$src2,$mask\t!" %}
22294 ins_encode %{
22295 int vlen_enc = vector_length_encoding(this);
22296 __ vblendvps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $mask$$XMMRegister, vlen_enc);
22297 %}
22298 ins_pipe( pipe_slow );
22299 %}
22300
22301 instruct vblendvp(legVec dst, legVec src1, legVec src2, legVec mask, legVec vtmp) %{
22302 predicate(UseAVX > 0 && EnableX86ECoreOpts &&
22303 n->in(2)->bottom_type()->isa_pvectmask() == nullptr &&
22304 Matcher::vector_length_in_bytes(n) <= 32);
22305 match(Set dst (VectorBlend (Binary src1 src2) mask));
22306 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using $vtmp as TEMP" %}
22307 effect(TEMP vtmp, TEMP dst);
22308 ins_encode %{
22309 int vlen_enc = vector_length_encoding(this);
22310 __ vpandn($vtmp$$XMMRegister, $mask$$XMMRegister, $src1$$XMMRegister, vlen_enc);
22311 __ vpand ($dst$$XMMRegister, $mask$$XMMRegister, $src2$$XMMRegister, vlen_enc);
22312 __ vpor ($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22313 %}
22314 ins_pipe( pipe_slow );
22315 %}
22316
22317 instruct evblendvp64(vec dst, vec src1, vec src2, vec mask, kReg ktmp) %{
22318 predicate(Matcher::vector_length_in_bytes(n) == 64 &&
22319 n->in(2)->bottom_type()->isa_pvectmask() == nullptr);
22320 match(Set dst (VectorBlend (Binary src1 src2) mask));
22321 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
22322 effect(TEMP ktmp);
22323 ins_encode %{
22324 int vlen_enc = Assembler::AVX_512bit;
22325 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22326 __ evpcmp(elem_bt, $ktmp$$KRegister, k0, $mask$$XMMRegister, ExternalAddress(vector_all_bits_set()), Assembler::eq, vlen_enc, noreg);
22327 __ evpblend(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
22328 %}
22329 ins_pipe( pipe_slow );
22330 %}
22331
22332
22333 instruct evblendvp64_masked(vec dst, vec src1, vec src2, kReg mask) %{
22334 predicate(n->in(2)->bottom_type()->isa_pvectmask() &&
22335 (!is_subword_type(Matcher::vector_element_basic_type(n)) ||
22336 VM_Version::supports_avx512bw()));
22337 match(Set dst (VectorBlend (Binary src1 src2) mask));
22338 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
22339 ins_encode %{
22340 int vlen_enc = vector_length_encoding(this);
22341 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22342 __ evpblend(elem_bt, $dst$$XMMRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
22343 %}
22344 ins_pipe( pipe_slow );
22345 %}
22346
22347 // --------------------------------- ABS --------------------------------------
22348 // a = |a|
22349 instruct vabsB_reg(vec dst, vec src) %{
22350 match(Set dst (AbsVB src));
22351 format %{ "vabsb $dst,$src\t# $dst = |$src| abs packedB" %}
22352 ins_encode %{
22353 uint vlen = Matcher::vector_length(this);
22354 if (vlen <= 16) {
22355 __ pabsb($dst$$XMMRegister, $src$$XMMRegister);
22356 } else {
22357 int vlen_enc = vector_length_encoding(this);
22358 __ vpabsb($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22359 }
22360 %}
22361 ins_pipe( pipe_slow );
22362 %}
22363
22364 instruct vabsS_reg(vec dst, vec src) %{
22365 match(Set dst (AbsVS src));
22366 format %{ "vabsw $dst,$src\t# $dst = |$src| abs packedS" %}
22367 ins_encode %{
22368 uint vlen = Matcher::vector_length(this);
22369 if (vlen <= 8) {
22370 __ pabsw($dst$$XMMRegister, $src$$XMMRegister);
22371 } else {
22372 int vlen_enc = vector_length_encoding(this);
22373 __ vpabsw($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22374 }
22375 %}
22376 ins_pipe( pipe_slow );
22377 %}
22378
22379 instruct vabsI_reg(vec dst, vec src) %{
22380 match(Set dst (AbsVI src));
22381 format %{ "pabsd $dst,$src\t# $dst = |$src| abs packedI" %}
22382 ins_encode %{
22383 uint vlen = Matcher::vector_length(this);
22384 if (vlen <= 4) {
22385 __ pabsd($dst$$XMMRegister, $src$$XMMRegister);
22386 } else {
22387 int vlen_enc = vector_length_encoding(this);
22388 __ vpabsd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22389 }
22390 %}
22391 ins_pipe( pipe_slow );
22392 %}
22393
22394 instruct vabsL_reg(vec dst, vec src) %{
22395 match(Set dst (AbsVL src));
22396 format %{ "evpabsq $dst,$src\t# $dst = |$src| abs packedL" %}
22397 ins_encode %{
22398 assert(UseAVX > 2, "required");
22399 int vlen_enc = vector_length_encoding(this);
22400 if (!VM_Version::supports_avx512vl()) {
22401 vlen_enc = Assembler::AVX_512bit;
22402 }
22403 __ evpabsq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22404 %}
22405 ins_pipe( pipe_slow );
22406 %}
22407
22408 // --------------------------------- ABSNEG --------------------------------------
22409
22410 instruct vabsnegF(vec dst, vec src) %{
22411 predicate(Matcher::vector_length(n) != 4); // handled by 1-operand instruction vabsneg4F
22412 match(Set dst (AbsVF src));
22413 match(Set dst (NegVF src));
22414 format %{ "vabsnegf $dst,$src,[mask]\t# absneg packedF" %}
22415 ins_cost(150);
22416 ins_encode %{
22417 int opcode = this->ideal_Opcode();
22418 int vlen = Matcher::vector_length(this);
22419 if (vlen == 2) {
22420 __ vabsnegf(opcode, $dst$$XMMRegister, $src$$XMMRegister);
22421 } else {
22422 assert(vlen == 8 || vlen == 16, "required");
22423 int vlen_enc = vector_length_encoding(this);
22424 __ vabsnegf(opcode, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22425 }
22426 %}
22427 ins_pipe( pipe_slow );
22428 %}
22429
22430 instruct vabsneg4F(vec dst) %{
22431 predicate(Matcher::vector_length(n) == 4);
22432 match(Set dst (AbsVF dst));
22433 match(Set dst (NegVF dst));
22434 format %{ "vabsnegf $dst,[mask]\t# absneg packed4F" %}
22435 ins_cost(150);
22436 ins_encode %{
22437 int opcode = this->ideal_Opcode();
22438 __ vabsnegf(opcode, $dst$$XMMRegister, $dst$$XMMRegister);
22439 %}
22440 ins_pipe( pipe_slow );
22441 %}
22442
22443 instruct vabsnegD(vec dst, vec src) %{
22444 match(Set dst (AbsVD src));
22445 match(Set dst (NegVD src));
22446 format %{ "vabsnegd $dst,$src,[mask]\t# absneg packedD" %}
22447 ins_encode %{
22448 int opcode = this->ideal_Opcode();
22449 uint vlen = Matcher::vector_length(this);
22450 if (vlen == 2) {
22451 __ vabsnegd(opcode, $dst$$XMMRegister, $src$$XMMRegister);
22452 } else {
22453 int vlen_enc = vector_length_encoding(this);
22454 __ vabsnegd(opcode, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22455 }
22456 %}
22457 ins_pipe( pipe_slow );
22458 %}
22459
22460 //------------------------------------- VectorTest --------------------------------------------
22461
22462 instruct vptest_lt16(rFlagsRegU cr, legVec src1, legVec src2, legVec vtmp) %{
22463 predicate(Matcher::vector_length_in_bytes(n->in(1)) < 16);
22464 match(Set cr (VectorTest src1 src2));
22465 effect(TEMP vtmp);
22466 format %{ "vptest_lt16 $src1, $src2\t! using $vtmp as TEMP" %}
22467 ins_encode %{
22468 BasicType bt = Matcher::vector_element_basic_type(this, $src1);
22469 int vlen = Matcher::vector_length_in_bytes(this, $src1);
22470 __ vectortest(bt, $src1$$XMMRegister, $src2$$XMMRegister, $vtmp$$XMMRegister, vlen);
22471 %}
22472 ins_pipe( pipe_slow );
22473 %}
22474
22475 instruct vptest_ge16(rFlagsRegU cr, legVec src1, legVec src2) %{
22476 predicate(Matcher::vector_length_in_bytes(n->in(1)) >= 16);
22477 match(Set cr (VectorTest src1 src2));
22478 format %{ "vptest_ge16 $src1, $src2\n\t" %}
22479 ins_encode %{
22480 BasicType bt = Matcher::vector_element_basic_type(this, $src1);
22481 int vlen = Matcher::vector_length_in_bytes(this, $src1);
22482 __ vectortest(bt, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, vlen);
22483 %}
22484 ins_pipe( pipe_slow );
22485 %}
22486
22487 instruct ktest_alltrue_le8(rFlagsRegU cr, kReg src1, kReg src2, rRegI tmp) %{
22488 predicate((Matcher::vector_length(n->in(1)) < 8 ||
22489 (Matcher::vector_length(n->in(1)) == 8 && !VM_Version::supports_avx512dq())) &&
22490 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
22491 match(Set cr (VectorTest src1 src2));
22492 effect(TEMP tmp);
22493 format %{ "ktest_alltrue_le8 $src1, $src2\t! using $tmp as TEMP" %}
22494 ins_encode %{
22495 uint masklen = Matcher::vector_length(this, $src1);
22496 __ kmovwl($tmp$$Register, $src1$$KRegister);
22497 __ andl($tmp$$Register, (1 << masklen) - 1);
22498 __ cmpl($tmp$$Register, (1 << masklen) - 1);
22499 %}
22500 ins_pipe( pipe_slow );
22501 %}
22502
22503 instruct ktest_anytrue_le8(rFlagsRegU cr, kReg src1, kReg src2, rRegI tmp) %{
22504 predicate((Matcher::vector_length(n->in(1)) < 8 ||
22505 (Matcher::vector_length(n->in(1)) == 8 && !VM_Version::supports_avx512dq())) &&
22506 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
22507 match(Set cr (VectorTest src1 src2));
22508 effect(TEMP tmp);
22509 format %{ "ktest_anytrue_le8 $src1, $src2\t! using $tmp as TEMP" %}
22510 ins_encode %{
22511 uint masklen = Matcher::vector_length(this, $src1);
22512 __ kmovwl($tmp$$Register, $src1$$KRegister);
22513 __ andl($tmp$$Register, (1 << masklen) - 1);
22514 %}
22515 ins_pipe( pipe_slow );
22516 %}
22517
22518 instruct ktest_ge8(rFlagsRegU cr, kReg src1, kReg src2) %{
22519 predicate(Matcher::vector_length(n->in(1)) >= 16 ||
22520 (Matcher::vector_length(n->in(1)) == 8 && VM_Version::supports_avx512dq()));
22521 match(Set cr (VectorTest src1 src2));
22522 format %{ "ktest_ge8 $src1, $src2\n\t" %}
22523 ins_encode %{
22524 uint masklen = Matcher::vector_length(this, $src1);
22525 __ kortest(masklen, $src1$$KRegister, $src1$$KRegister);
22526 %}
22527 ins_pipe( pipe_slow );
22528 %}
22529
22530 //------------------------------------- LoadMask --------------------------------------------
22531
22532 instruct loadMask(legVec dst, legVec src) %{
22533 predicate(n->bottom_type()->isa_pvectmask() == nullptr && !VM_Version::supports_avx512vlbw());
22534 match(Set dst (VectorLoadMask src));
22535 effect(TEMP dst);
22536 format %{ "vector_loadmask_byte $dst, $src\n\t" %}
22537 ins_encode %{
22538 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
22539 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22540 __ load_vector_mask($dst$$XMMRegister, $src$$XMMRegister, vlen_in_bytes, elem_bt, true);
22541 %}
22542 ins_pipe( pipe_slow );
22543 %}
22544
22545 instruct loadMask64(kReg dst, vec src, vec xtmp) %{
22546 predicate(n->bottom_type()->isa_pvectmask() && !VM_Version::supports_avx512vlbw());
22547 match(Set dst (VectorLoadMask src));
22548 effect(TEMP xtmp);
22549 format %{ "vector_loadmask_64byte $dst, $src\t! using $xtmp as TEMP" %}
22550 ins_encode %{
22551 __ load_vector_mask($dst$$KRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
22552 true, Assembler::AVX_512bit);
22553 %}
22554 ins_pipe( pipe_slow );
22555 %}
22556
22557 instruct loadMask_evex(kReg dst, vec src, vec xtmp) %{
22558 predicate(n->bottom_type()->isa_pvectmask() && VM_Version::supports_avx512vlbw());
22559 match(Set dst (VectorLoadMask src));
22560 effect(TEMP xtmp);
22561 format %{ "vector_loadmask_byte $dst, $src\t! using $xtmp as TEMP" %}
22562 ins_encode %{
22563 int vlen_enc = vector_length_encoding(in(1));
22564 __ load_vector_mask($dst$$KRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
22565 false, vlen_enc);
22566 %}
22567 ins_pipe( pipe_slow );
22568 %}
22569
22570 //------------------------------------- StoreMask --------------------------------------------
22571
22572 instruct vstoreMask1B(vec dst, vec src, immI_1 size) %{
22573 predicate(Matcher::vector_length(n) < 64 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22574 match(Set dst (VectorStoreMask src size));
22575 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22576 ins_encode %{
22577 int vlen = Matcher::vector_length(this);
22578 if (vlen <= 16 && UseAVX <= 2) {
22579 assert(UseSSE >= 3, "required");
22580 __ pabsb($dst$$XMMRegister, $src$$XMMRegister);
22581 } else {
22582 assert(UseAVX > 0, "required");
22583 int src_vlen_enc = vector_length_encoding(this, $src);
22584 __ vpabsb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
22585 }
22586 %}
22587 ins_pipe( pipe_slow );
22588 %}
22589
22590 instruct vstoreMask2B(vec dst, vec src, vec xtmp, immI_2 size) %{
22591 predicate(Matcher::vector_length(n) <= 16 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22592 match(Set dst (VectorStoreMask src size));
22593 effect(TEMP_DEF dst, TEMP xtmp);
22594 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22595 ins_encode %{
22596 int vlen_enc = Assembler::AVX_128bit;
22597 int vlen = Matcher::vector_length(this);
22598 if (vlen <= 8) {
22599 assert(UseSSE >= 3, "required");
22600 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
22601 __ pabsw($dst$$XMMRegister, $src$$XMMRegister);
22602 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
22603 } else {
22604 assert(UseAVX > 0, "required");
22605 __ vextracti128($dst$$XMMRegister, $src$$XMMRegister, 0x1);
22606 __ vpacksswb($dst$$XMMRegister, $src$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22607 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22608 }
22609 %}
22610 ins_pipe( pipe_slow );
22611 %}
22612
22613 instruct vstoreMask4B(vec dst, vec src, vec xtmp, immI_4 size) %{
22614 predicate(UseAVX <= 2 && Matcher::vector_length(n) <= 8 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22615 match(Set dst (VectorStoreMask src size));
22616 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22617 effect(TEMP_DEF dst, TEMP xtmp);
22618 ins_encode %{
22619 int vlen_enc = Assembler::AVX_128bit;
22620 int vlen = Matcher::vector_length(this);
22621 if (vlen <= 4) {
22622 assert(UseSSE >= 3, "required");
22623 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
22624 __ pabsd($dst$$XMMRegister, $src$$XMMRegister);
22625 __ packusdw($dst$$XMMRegister, $xtmp$$XMMRegister);
22626 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
22627 } else {
22628 assert(UseAVX > 0, "required");
22629 __ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
22630 __ vextracti128($dst$$XMMRegister, $src$$XMMRegister, 0x1);
22631 __ vpackssdw($dst$$XMMRegister, $src$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22632 __ vpacksswb($dst$$XMMRegister, $dst$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
22633 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22634 }
22635 %}
22636 ins_pipe( pipe_slow );
22637 %}
22638
22639 instruct storeMask8B(vec dst, vec src, vec xtmp, immI_8 size) %{
22640 predicate(UseAVX <= 2 && Matcher::vector_length(n) == 2);
22641 match(Set dst (VectorStoreMask src size));
22642 effect(TEMP_DEF dst, TEMP xtmp);
22643 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22644 ins_encode %{
22645 assert(UseSSE >= 3, "required");
22646 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
22647 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x8);
22648 __ pabsd($dst$$XMMRegister, $dst$$XMMRegister);
22649 __ packusdw($dst$$XMMRegister, $xtmp$$XMMRegister);
22650 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
22651 %}
22652 ins_pipe( pipe_slow );
22653 %}
22654
22655 instruct storeMask8B_avx(vec dst, vec src, immI_8 size, vec vtmp) %{
22656 predicate(UseAVX <= 2 && Matcher::vector_length(n) == 4);
22657 match(Set dst (VectorStoreMask src size));
22658 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s], using $vtmp as TEMP" %}
22659 effect(TEMP_DEF dst, TEMP vtmp);
22660 ins_encode %{
22661 int vlen_enc = Assembler::AVX_128bit;
22662 __ vshufps($dst$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 0x88, Assembler::AVX_256bit);
22663 __ vextracti128($vtmp$$XMMRegister, $dst$$XMMRegister, 0x1);
22664 __ vblendps($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0xC, vlen_enc);
22665 __ vpxor($vtmp$$XMMRegister, $vtmp$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22666 __ vpackssdw($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22667 __ vpacksswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22668 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22669 %}
22670 ins_pipe( pipe_slow );
22671 %}
22672
22673 instruct vstoreMask4B_evex_novectmask(vec dst, vec src, immI_4 size) %{
22674 predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22675 match(Set dst (VectorStoreMask src size));
22676 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22677 ins_encode %{
22678 int src_vlen_enc = vector_length_encoding(this, $src);
22679 int dst_vlen_enc = vector_length_encoding(this);
22680 if (!VM_Version::supports_avx512vl()) {
22681 src_vlen_enc = Assembler::AVX_512bit;
22682 }
22683 __ evpmovdb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
22684 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
22685 %}
22686 ins_pipe( pipe_slow );
22687 %}
22688
22689 instruct vstoreMask8B_evex_novectmask(vec dst, vec src, immI_8 size) %{
22690 predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22691 match(Set dst (VectorStoreMask src size));
22692 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22693 ins_encode %{
22694 int src_vlen_enc = vector_length_encoding(this, $src);
22695 int dst_vlen_enc = vector_length_encoding(this);
22696 if (!VM_Version::supports_avx512vl()) {
22697 src_vlen_enc = Assembler::AVX_512bit;
22698 }
22699 __ evpmovqb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
22700 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
22701 %}
22702 ins_pipe( pipe_slow );
22703 %}
22704
22705 instruct vstoreMask_evex_vectmask(vec dst, kReg mask, immI size) %{
22706 predicate(n->in(1)->bottom_type()->isa_pvectmask() && !VM_Version::supports_avx512vlbw());
22707 match(Set dst (VectorStoreMask mask size));
22708 effect(TEMP_DEF dst);
22709 format %{ "vector_store_mask $dst, $mask \t! elem size is $size byte[s]" %}
22710 ins_encode %{
22711 assert(Matcher::vector_length_in_bytes(this, $mask) == 64, "");
22712 __ evmovdqul($dst$$XMMRegister, $mask$$KRegister, ExternalAddress(vector_int_mask_cmp_bits()),
22713 false, Assembler::AVX_512bit, noreg);
22714 __ evpmovdb($dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_512bit);
22715 %}
22716 ins_pipe( pipe_slow );
22717 %}
22718
22719 instruct vstoreMask_evex(vec dst, kReg mask, immI size) %{
22720 predicate(n->in(1)->bottom_type()->isa_pvectmask() && VM_Version::supports_avx512vlbw());
22721 match(Set dst (VectorStoreMask mask size));
22722 effect(TEMP_DEF dst);
22723 format %{ "vector_store_mask $dst, $mask \t! elem size is $size byte[s]" %}
22724 ins_encode %{
22725 int dst_vlen_enc = vector_length_encoding(this);
22726 __ evpmovm2b($dst$$XMMRegister, $mask$$KRegister, dst_vlen_enc);
22727 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
22728 %}
22729 ins_pipe( pipe_slow );
22730 %}
22731
22732 instruct vmaskcast_evex(kReg dst) %{
22733 match(Set dst (VectorMaskCast dst));
22734 ins_cost(0);
22735 format %{ "vector_mask_cast $dst" %}
22736 ins_encode %{
22737 // empty
22738 %}
22739 ins_pipe(empty);
22740 %}
22741
22742 instruct vmaskcast(vec dst) %{
22743 predicate(Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
22744 match(Set dst (VectorMaskCast dst));
22745 ins_cost(0);
22746 format %{ "vector_mask_cast $dst" %}
22747 ins_encode %{
22748 // empty
22749 %}
22750 ins_pipe(empty);
22751 %}
22752
22753 instruct vmaskcast_avx(vec dst, vec src) %{
22754 predicate(Matcher::vector_length_in_bytes(n) != Matcher::vector_length_in_bytes(n->in(1)));
22755 match(Set dst (VectorMaskCast src));
22756 format %{ "vector_mask_cast $dst, $src" %}
22757 ins_encode %{
22758 int vlen = Matcher::vector_length(this);
22759 BasicType src_bt = Matcher::vector_element_basic_type(this, $src);
22760 BasicType dst_bt = Matcher::vector_element_basic_type(this);
22761 __ vector_mask_cast($dst$$XMMRegister, $src$$XMMRegister, dst_bt, src_bt, vlen);
22762 %}
22763 ins_pipe(pipe_slow);
22764 %}
22765
22766 //-------------------------------- Load Iota Indices ----------------------------------
22767
22768 instruct loadIotaIndices(vec dst, immI_0 src) %{
22769 match(Set dst (VectorLoadConst src));
22770 format %{ "vector_load_iota $dst CONSTANT_MEMORY\t! load iota indices" %}
22771 ins_encode %{
22772 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
22773 BasicType bt = Matcher::vector_element_basic_type(this);
22774 __ load_iota_indices($dst$$XMMRegister, vlen_in_bytes, bt);
22775 %}
22776 ins_pipe( pipe_slow );
22777 %}
22778
22779 instruct VectorPopulateIndex(vec dst, rRegI src1, immI_1 src2, vec vtmp) %{
22780 match(Set dst (PopulateIndex src1 src2));
22781 effect(TEMP dst, TEMP vtmp);
22782 format %{ "vector_populate_index $dst $src1 $src2\t! using $vtmp as TEMP" %}
22783 ins_encode %{
22784 assert($src2$$constant == 1, "required");
22785 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
22786 int vlen_enc = vector_length_encoding(this);
22787 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22788 __ vpbroadcast(elem_bt, $vtmp$$XMMRegister, $src1$$Register, vlen_enc);
22789 __ load_iota_indices($dst$$XMMRegister, vlen_in_bytes, elem_bt);
22790 __ vpadd(elem_bt, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22791 %}
22792 ins_pipe( pipe_slow );
22793 %}
22794
22795 instruct VectorPopulateLIndex(vec dst, rRegL src1, immI_1 src2, vec vtmp) %{
22796 match(Set dst (PopulateIndex src1 src2));
22797 effect(TEMP dst, TEMP vtmp);
22798 format %{ "vector_populate_index $dst $src1 $src2\t! using $vtmp as TEMP" %}
22799 ins_encode %{
22800 assert($src2$$constant == 1, "required");
22801 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
22802 int vlen_enc = vector_length_encoding(this);
22803 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22804 __ vpbroadcast(elem_bt, $vtmp$$XMMRegister, $src1$$Register, vlen_enc);
22805 __ load_iota_indices($dst$$XMMRegister, vlen_in_bytes, elem_bt);
22806 __ vpadd(elem_bt, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22807 %}
22808 ins_pipe( pipe_slow );
22809 %}
22810
22811 //-------------------------------- Rearrange ----------------------------------
22812
22813 // LoadShuffle/Rearrange for Byte
22814 instruct rearrangeB(vec dst, vec shuffle) %{
22815 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
22816 Matcher::vector_length(n) < 32);
22817 match(Set dst (VectorRearrange dst shuffle));
22818 format %{ "vector_rearrange $dst, $shuffle, $dst" %}
22819 ins_encode %{
22820 assert(UseSSE >= 4, "required");
22821 __ pshufb($dst$$XMMRegister, $shuffle$$XMMRegister);
22822 %}
22823 ins_pipe( pipe_slow );
22824 %}
22825
22826 instruct rearrangeB_avx(legVec dst, legVec src, vec shuffle, legVec vtmp1, legVec vtmp2) %{
22827 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
22828 Matcher::vector_length(n) == 32 && !VM_Version::supports_avx512_vbmi());
22829 match(Set dst (VectorRearrange src shuffle));
22830 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
22831 format %{ "vector_rearrange $dst, $shuffle, $src\t! using $vtmp1, $vtmp2 as TEMP" %}
22832 ins_encode %{
22833 assert(UseAVX >= 2, "required");
22834 // Swap src into vtmp1
22835 __ vperm2i128($vtmp1$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 1);
22836 // Shuffle swapped src to get entries from other 128 bit lane
22837 __ vpshufb($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
22838 // Shuffle original src to get entries from self 128 bit lane
22839 __ vpshufb($dst$$XMMRegister, $src$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
22840 // Create a blend mask by setting high bits for entries coming from other lane in shuffle
22841 __ vpaddb($vtmp2$$XMMRegister, $shuffle$$XMMRegister, ExternalAddress(vector_byte_shufflemask()), Assembler::AVX_256bit, noreg);
22842 // Perform the blend
22843 __ vpblendvb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, Assembler::AVX_256bit);
22844 %}
22845 ins_pipe( pipe_slow );
22846 %}
22847
22848
22849 instruct rearrangeB_evex(vec dst, vec src, vec shuffle, vec xtmp1, vec xtmp2, vec xtmp3, kReg ktmp, rRegI rtmp) %{
22850 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
22851 Matcher::vector_length(n) > 32 && !VM_Version::supports_avx512_vbmi());
22852 match(Set dst (VectorRearrange src shuffle));
22853 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP ktmp, TEMP rtmp);
22854 format %{ "vector_rearrange $dst, $shuffle, $src!\t using $xtmp1, $xtmp2, $xtmp3, $rtmp and $ktmp as TEMP" %}
22855 ins_encode %{
22856 int vlen_enc = vector_length_encoding(this);
22857 __ rearrange_bytes($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister,
22858 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $xtmp3$$XMMRegister,
22859 $rtmp$$Register, $ktmp$$KRegister, vlen_enc);
22860 %}
22861 ins_pipe( pipe_slow );
22862 %}
22863
22864 instruct rearrangeB_evex_vbmi(vec dst, vec src, vec shuffle) %{
22865 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
22866 Matcher::vector_length(n) >= 32 && VM_Version::supports_avx512_vbmi());
22867 match(Set dst (VectorRearrange src shuffle));
22868 format %{ "vector_rearrange $dst, $shuffle, $src" %}
22869 ins_encode %{
22870 int vlen_enc = vector_length_encoding(this);
22871 __ vpermb($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
22872 %}
22873 ins_pipe( pipe_slow );
22874 %}
22875
22876 // LoadShuffle/Rearrange for Short
22877
22878 instruct loadShuffleS(vec dst, vec src, vec vtmp) %{
22879 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
22880 !VM_Version::supports_avx512bw());
22881 match(Set dst (VectorLoadShuffle src));
22882 effect(TEMP dst, TEMP vtmp);
22883 format %{ "vector_load_shuffle $dst, $src\t! using $vtmp as TEMP" %}
22884 ins_encode %{
22885 // Create a byte shuffle mask from short shuffle mask
22886 // only byte shuffle instruction available on these platforms
22887 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
22888 if (UseAVX == 0) {
22889 assert(vlen_in_bytes <= 16, "required");
22890 // Multiply each shuffle by two to get byte index
22891 __ movdqu($vtmp$$XMMRegister, $src$$XMMRegister);
22892 __ psllw($vtmp$$XMMRegister, 1);
22893
22894 // Duplicate to create 2 copies of byte index
22895 __ movdqu($dst$$XMMRegister, $vtmp$$XMMRegister);
22896 __ psllw($dst$$XMMRegister, 8);
22897 __ por($dst$$XMMRegister, $vtmp$$XMMRegister);
22898
22899 // Add one to get alternate byte index
22900 __ movdqu($vtmp$$XMMRegister, ExternalAddress(vector_short_shufflemask()), noreg);
22901 __ paddb($dst$$XMMRegister, $vtmp$$XMMRegister);
22902 } else {
22903 assert(UseAVX > 1 || vlen_in_bytes <= 16, "required");
22904 int vlen_enc = vector_length_encoding(this);
22905 // Multiply each shuffle by two to get byte index
22906 __ vpsllw($vtmp$$XMMRegister, $src$$XMMRegister, 1, vlen_enc);
22907
22908 // Duplicate to create 2 copies of byte index
22909 __ vpsllw($dst$$XMMRegister, $vtmp$$XMMRegister, 8, vlen_enc);
22910 __ vpor($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22911
22912 // Add one to get alternate byte index
22913 __ vpaddb($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_short_shufflemask()), vlen_enc, noreg);
22914 }
22915 %}
22916 ins_pipe( pipe_slow );
22917 %}
22918
22919 instruct rearrangeS(vec dst, vec shuffle) %{
22920 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
22921 Matcher::vector_length(n) <= 8 && !VM_Version::supports_avx512bw());
22922 match(Set dst (VectorRearrange dst shuffle));
22923 format %{ "vector_rearrange $dst, $shuffle, $dst" %}
22924 ins_encode %{
22925 assert(UseSSE >= 4, "required");
22926 __ pshufb($dst$$XMMRegister, $shuffle$$XMMRegister);
22927 %}
22928 ins_pipe( pipe_slow );
22929 %}
22930
22931 instruct rearrangeS_avx(legVec dst, legVec src, vec shuffle, legVec vtmp1, legVec vtmp2) %{
22932 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
22933 Matcher::vector_length(n) == 16 && !VM_Version::supports_avx512bw());
22934 match(Set dst (VectorRearrange src shuffle));
22935 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
22936 format %{ "vector_rearrange $dst, $shuffle, $src\t! using $vtmp1, $vtmp2 as TEMP" %}
22937 ins_encode %{
22938 assert(UseAVX >= 2, "required");
22939 // Swap src into vtmp1
22940 __ vperm2i128($vtmp1$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 1);
22941 // Shuffle swapped src to get entries from other 128 bit lane
22942 __ vpshufb($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
22943 // Shuffle original src to get entries from self 128 bit lane
22944 __ vpshufb($dst$$XMMRegister, $src$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
22945 // Create a blend mask by setting high bits for entries coming from other lane in shuffle
22946 __ vpaddb($vtmp2$$XMMRegister, $shuffle$$XMMRegister, ExternalAddress(vector_byte_shufflemask()), Assembler::AVX_256bit, noreg);
22947 // Perform the blend
22948 __ vpblendvb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, Assembler::AVX_256bit);
22949 %}
22950 ins_pipe( pipe_slow );
22951 %}
22952
22953 instruct rearrangeS_evex(vec dst, vec src, vec shuffle) %{
22954 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
22955 VM_Version::supports_avx512bw());
22956 match(Set dst (VectorRearrange src shuffle));
22957 format %{ "vector_rearrange $dst, $shuffle, $src" %}
22958 ins_encode %{
22959 int vlen_enc = vector_length_encoding(this);
22960 if (!VM_Version::supports_avx512vl()) {
22961 vlen_enc = Assembler::AVX_512bit;
22962 }
22963 __ vpermw($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
22964 %}
22965 ins_pipe( pipe_slow );
22966 %}
22967
22968 // LoadShuffle/Rearrange for Integer and Float
22969
22970 instruct loadShuffleI(vec dst, vec src, vec vtmp) %{
22971 predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
22972 Matcher::vector_length(n) == 4 && UseAVX == 0);
22973 match(Set dst (VectorLoadShuffle src));
22974 effect(TEMP dst, TEMP vtmp);
22975 format %{ "vector_load_shuffle $dst, $src\t! using $vtmp as TEMP" %}
22976 ins_encode %{
22977 assert(UseSSE >= 4, "required");
22978
22979 // Create a byte shuffle mask from int shuffle mask
22980 // only byte shuffle instruction available on these platforms
22981
22982 // Duplicate and multiply each shuffle by 4
22983 __ movdqu($vtmp$$XMMRegister, $src$$XMMRegister);
22984 __ pshuflw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 0xA0);
22985 __ pshufhw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 0xA0);
22986 __ psllw($vtmp$$XMMRegister, 2);
22987
22988 // Duplicate again to create 4 copies of byte index
22989 __ movdqu($dst$$XMMRegister, $vtmp$$XMMRegister);
22990 __ psllw($dst$$XMMRegister, 8);
22991 __ por($vtmp$$XMMRegister, $dst$$XMMRegister);
22992
22993 // Add 3,2,1,0 to get alternate byte index
22994 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_int_shufflemask()), noreg);
22995 __ paddb($dst$$XMMRegister, $vtmp$$XMMRegister);
22996 %}
22997 ins_pipe( pipe_slow );
22998 %}
22999
23000 instruct rearrangeI(vec dst, vec shuffle) %{
23001 predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
23002 UseAVX == 0);
23003 match(Set dst (VectorRearrange dst shuffle));
23004 format %{ "vector_rearrange $dst, $shuffle, $dst" %}
23005 ins_encode %{
23006 assert(UseSSE >= 4, "required");
23007 __ pshufb($dst$$XMMRegister, $shuffle$$XMMRegister);
23008 %}
23009 ins_pipe( pipe_slow );
23010 %}
23011
23012 instruct rearrangeI_avx(vec dst, vec src, vec shuffle) %{
23013 predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
23014 UseAVX > 0);
23015 match(Set dst (VectorRearrange src shuffle));
23016 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23017 ins_encode %{
23018 int vlen_enc = vector_length_encoding(this);
23019 BasicType bt = Matcher::vector_element_basic_type(this);
23020 __ vector_rearrange_int_float(bt, $dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23021 %}
23022 ins_pipe( pipe_slow );
23023 %}
23024
23025 // LoadShuffle/Rearrange for Long and Double
23026
23027 instruct loadShuffleL(vec dst, vec src, vec vtmp) %{
23028 predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
23029 Matcher::vector_length(n) < 8 && !VM_Version::supports_avx512vl());
23030 match(Set dst (VectorLoadShuffle src));
23031 effect(TEMP dst, TEMP vtmp);
23032 format %{ "vector_load_shuffle $dst, $src\t! using $vtmp as TEMP" %}
23033 ins_encode %{
23034 assert(UseAVX >= 2, "required");
23035
23036 int vlen_enc = vector_length_encoding(this);
23037 // Create a double word shuffle mask from long shuffle mask
23038 // only double word shuffle instruction available on these platforms
23039
23040 // Multiply each shuffle by two to get double word index
23041 __ vpsllq($vtmp$$XMMRegister, $src$$XMMRegister, 1, vlen_enc);
23042
23043 // Duplicate each double word shuffle
23044 __ vpsllq($dst$$XMMRegister, $vtmp$$XMMRegister, 32, vlen_enc);
23045 __ vpor($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
23046
23047 // Add one to get alternate double word index
23048 __ vpaddd($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_long_shufflemask()), vlen_enc, noreg);
23049 %}
23050 ins_pipe( pipe_slow );
23051 %}
23052
23053 instruct rearrangeL(vec dst, vec src, vec shuffle) %{
23054 predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
23055 Matcher::vector_length(n) < 8 && !VM_Version::supports_avx512vl());
23056 match(Set dst (VectorRearrange src shuffle));
23057 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23058 ins_encode %{
23059 assert(UseAVX >= 2, "required");
23060
23061 int vlen_enc = vector_length_encoding(this);
23062 __ vpermd($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23063 %}
23064 ins_pipe( pipe_slow );
23065 %}
23066
23067 instruct rearrangeL_evex(vec dst, vec src, vec shuffle) %{
23068 predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
23069 (Matcher::vector_length(n) == 8 || VM_Version::supports_avx512vl()));
23070 match(Set dst (VectorRearrange src shuffle));
23071 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23072 ins_encode %{
23073 assert(UseAVX > 2, "required");
23074
23075 int vlen_enc = vector_length_encoding(this);
23076 if (vlen_enc == Assembler::AVX_128bit) {
23077 vlen_enc = Assembler::AVX_256bit;
23078 }
23079 __ vpermq($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23080 %}
23081 ins_pipe( pipe_slow );
23082 %}
23083
23084 // --------------------------------- FMA --------------------------------------
23085 // a * b + c
23086
23087 instruct vfmaF_reg(vec a, vec b, vec c) %{
23088 match(Set c (FmaVF c (Binary a b)));
23089 format %{ "fmaps $a,$b,$c\t# $c = $a * $b + $c fma packedF" %}
23090 ins_cost(150);
23091 ins_encode %{
23092 assert(UseFMA, "not enabled");
23093 int vlen_enc = vector_length_encoding(this);
23094 __ vfmaf($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister, vlen_enc);
23095 %}
23096 ins_pipe( pipe_slow );
23097 %}
23098
23099 instruct vfmaF_mem(vec a, memory b, vec c) %{
23100 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
23101 match(Set c (FmaVF c (Binary a (LoadVector b))));
23102 format %{ "fmaps $a,$b,$c\t# $c = $a * $b + $c fma packedF" %}
23103 ins_cost(150);
23104 ins_encode %{
23105 assert(UseFMA, "not enabled");
23106 int vlen_enc = vector_length_encoding(this);
23107 __ vfmaf($c$$XMMRegister, $a$$XMMRegister, $b$$Address, $c$$XMMRegister, vlen_enc);
23108 %}
23109 ins_pipe( pipe_slow );
23110 %}
23111
23112 instruct vfmaD_reg(vec a, vec b, vec c) %{
23113 match(Set c (FmaVD c (Binary a b)));
23114 format %{ "fmapd $a,$b,$c\t# $c = $a * $b + $c fma packedD" %}
23115 ins_cost(150);
23116 ins_encode %{
23117 assert(UseFMA, "not enabled");
23118 int vlen_enc = vector_length_encoding(this);
23119 __ vfmad($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister, vlen_enc);
23120 %}
23121 ins_pipe( pipe_slow );
23122 %}
23123
23124 instruct vfmaD_mem(vec a, memory b, vec c) %{
23125 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
23126 match(Set c (FmaVD c (Binary a (LoadVector b))));
23127 format %{ "fmapd $a,$b,$c\t# $c = $a * $b + $c fma packedD" %}
23128 ins_cost(150);
23129 ins_encode %{
23130 assert(UseFMA, "not enabled");
23131 int vlen_enc = vector_length_encoding(this);
23132 __ vfmad($c$$XMMRegister, $a$$XMMRegister, $b$$Address, $c$$XMMRegister, vlen_enc);
23133 %}
23134 ins_pipe( pipe_slow );
23135 %}
23136
23137 // --------------------------------- Vector Multiply Add --------------------------------------
23138
23139 instruct vmuladdS2I_reg_sse(vec dst, vec src1) %{
23140 predicate(UseAVX == 0);
23141 match(Set dst (MulAddVS2VI dst src1));
23142 format %{ "pmaddwd $dst,$src1\t! muladd packedStoI" %}
23143 ins_encode %{
23144 __ pmaddwd($dst$$XMMRegister, $src1$$XMMRegister);
23145 %}
23146 ins_pipe( pipe_slow );
23147 %}
23148
23149 instruct vmuladdS2I_reg_avx(vec dst, vec src1, vec src2) %{
23150 predicate(UseAVX > 0);
23151 match(Set dst (MulAddVS2VI src1 src2));
23152 format %{ "vpmaddwd $dst,$src1,$src2\t! muladd packedStoI" %}
23153 ins_encode %{
23154 int vlen_enc = vector_length_encoding(this);
23155 __ vpmaddwd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
23156 %}
23157 ins_pipe( pipe_slow );
23158 %}
23159
23160 // --------------------------------- Vector Multiply Add Add ----------------------------------
23161
23162 instruct vmuladdaddS2I_reg(vec dst, vec src1, vec src2) %{
23163 predicate(VM_Version::supports_avx512_vnni());
23164 match(Set dst (AddVI (MulAddVS2VI src1 src2) dst));
23165 format %{ "evpdpwssd $dst,$src1,$src2\t! muladdadd packedStoI" %}
23166 ins_encode %{
23167 assert(UseAVX > 2, "required");
23168 int vlen_enc = vector_length_encoding(this);
23169 __ evpdpwssd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
23170 %}
23171 ins_pipe( pipe_slow );
23172 ins_cost(10);
23173 %}
23174
23175 // --------------------------------- PopCount --------------------------------------
23176
23177 instruct vpopcount_integral_reg_evex(vec dst, vec src) %{
23178 predicate(is_vector_popcount_predicate(Matcher::vector_element_basic_type(n->in(1))));
23179 match(Set dst (PopCountVI src));
23180 match(Set dst (PopCountVL src));
23181 format %{ "vector_popcount_integral $dst, $src" %}
23182 ins_encode %{
23183 int opcode = this->ideal_Opcode();
23184 int vlen_enc = vector_length_encoding(this, $src);
23185 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23186 __ vector_popcount_integral_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, k0, true, vlen_enc);
23187 %}
23188 ins_pipe( pipe_slow );
23189 %}
23190
23191 instruct vpopcount_integral_reg_evex_masked(vec dst, vec src, kReg mask) %{
23192 predicate(is_vector_popcount_predicate(Matcher::vector_element_basic_type(n->in(1))));
23193 match(Set dst (PopCountVI src mask));
23194 match(Set dst (PopCountVL src mask));
23195 format %{ "vector_popcount_integral_masked $dst, $src, $mask" %}
23196 ins_encode %{
23197 int vlen_enc = vector_length_encoding(this, $src);
23198 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23199 __ evmovdquq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
23200 __ vector_popcount_integral_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $mask$$KRegister, true, vlen_enc);
23201 %}
23202 ins_pipe( pipe_slow );
23203 %}
23204
23205 instruct vpopcount_avx_reg(vec dst, vec src, vec xtmp1, vec xtmp2, rRegP rtmp) %{
23206 predicate(!is_vector_popcount_predicate(Matcher::vector_element_basic_type(n->in(1))));
23207 match(Set dst (PopCountVI src));
23208 match(Set dst (PopCountVL src));
23209 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp);
23210 format %{ "vector_popcount_integral $dst, $src\t! using $xtmp1, $xtmp2, and $rtmp as TEMP" %}
23211 ins_encode %{
23212 int opcode = this->ideal_Opcode();
23213 int vlen_enc = vector_length_encoding(this, $src);
23214 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23215 __ vector_popcount_integral(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23216 $xtmp2$$XMMRegister, $rtmp$$Register, vlen_enc);
23217 %}
23218 ins_pipe( pipe_slow );
23219 %}
23220
23221 // --------------------------------- Vector Trailing Zeros Count --------------------------------------
23222
23223 instruct vcount_trailing_zeros_reg_evex(vec dst, vec src, vec xtmp, rRegP rtmp) %{
23224 predicate(is_clz_non_subword_predicate_evex(Matcher::vector_element_basic_type(n->in(1)),
23225 Matcher::vector_length_in_bytes(n->in(1))));
23226 match(Set dst (CountTrailingZerosV src));
23227 effect(TEMP dst, TEMP xtmp, TEMP rtmp);
23228 ins_cost(400);
23229 format %{ "vector_count_trailing_zeros $dst, $src!\t using $xtmp and $rtmp as TEMP" %}
23230 ins_encode %{
23231 int vlen_enc = vector_length_encoding(this, $src);
23232 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23233 __ vector_count_trailing_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, xnoreg,
23234 xnoreg, xnoreg, $xtmp$$XMMRegister, k0, $rtmp$$Register, vlen_enc);
23235 %}
23236 ins_pipe( pipe_slow );
23237 %}
23238
23239 instruct vcount_trailing_zeros_short_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, rRegP rtmp) %{
23240 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_SHORT &&
23241 VM_Version::supports_avx512cd() &&
23242 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64));
23243 match(Set dst (CountTrailingZerosV src));
23244 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp);
23245 ins_cost(400);
23246 format %{ "vector_count_trailing_zeros $dst, $src!\t using $xtmp1, $xtmp2, $xtmp3 and $rtmp as TEMP" %}
23247 ins_encode %{
23248 int vlen_enc = vector_length_encoding(this, $src);
23249 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23250 __ vector_count_trailing_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23251 $xtmp2$$XMMRegister, xnoreg, $xtmp3$$XMMRegister, k0, $rtmp$$Register, vlen_enc);
23252 %}
23253 ins_pipe( pipe_slow );
23254 %}
23255
23256 instruct vcount_trailing_zeros_byte_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, kReg ktmp, rRegP rtmp) %{
23257 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_BYTE && VM_Version::supports_avx512vlbw());
23258 match(Set dst (CountTrailingZerosV src));
23259 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, TEMP ktmp, TEMP rtmp);
23260 ins_cost(400);
23261 format %{ "vector_count_trailing_zeros $dst, $src!\t using $xtmp1, $xtmp2, $xtmp3, $xtmp4, $ktmp and $rtmp as TEMP" %}
23262 ins_encode %{
23263 int vlen_enc = vector_length_encoding(this, $src);
23264 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23265 __ vector_count_trailing_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23266 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister,
23267 $ktmp$$KRegister, $rtmp$$Register, vlen_enc);
23268 %}
23269 ins_pipe( pipe_slow );
23270 %}
23271
23272 instruct vcount_trailing_zeros_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, rRegP rtmp) %{
23273 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n->in(1)) < 64);
23274 match(Set dst (CountTrailingZerosV src));
23275 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp);
23276 format %{ "vector_count_trailing_zeros $dst, $src\t! using $xtmp1, $xtmp2, $xtmp3, and $rtmp as TEMP" %}
23277 ins_encode %{
23278 int vlen_enc = vector_length_encoding(this, $src);
23279 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23280 __ vector_count_trailing_zeros_avx(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23281 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, vlen_enc);
23282 %}
23283 ins_pipe( pipe_slow );
23284 %}
23285
23286
23287 // --------------------------------- Bitwise Ternary Logic ----------------------------------
23288
23289 instruct vpternlog(vec dst, vec src2, vec src3, immU8 func) %{
23290 match(Set dst (MacroLogicV (Binary dst src2) (Binary src3 func)));
23291 effect(TEMP dst);
23292 format %{ "vpternlogd $dst,$src2,$src3,$func\t! vector ternary logic" %}
23293 ins_encode %{
23294 int vector_len = vector_length_encoding(this);
23295 __ vpternlogd($dst$$XMMRegister, $func$$constant, $src2$$XMMRegister, $src3$$XMMRegister, vector_len);
23296 %}
23297 ins_pipe( pipe_slow );
23298 %}
23299
23300 instruct vpternlog_mem(vec dst, vec src2, memory src3, immU8 func) %{
23301 predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) > 8);
23302 match(Set dst (MacroLogicV (Binary dst src2) (Binary (LoadVector src3) func)));
23303 effect(TEMP dst);
23304 format %{ "vpternlogd $dst,$src2,$src3,$func\t! vector ternary logic" %}
23305 ins_encode %{
23306 int vector_len = vector_length_encoding(this);
23307 __ vpternlogd($dst$$XMMRegister, $func$$constant, $src2$$XMMRegister, $src3$$Address, vector_len);
23308 %}
23309 ins_pipe( pipe_slow );
23310 %}
23311
23312 // --------------------------------- Rotation Operations ----------------------------------
23313 instruct vprotate_immI8(vec dst, vec src, immI8 shift) %{
23314 match(Set dst (RotateLeftV src shift));
23315 match(Set dst (RotateRightV src shift));
23316 format %{ "vprotate_imm8 $dst,$src,$shift\t! vector rotate" %}
23317 ins_encode %{
23318 int opcode = this->ideal_Opcode();
23319 int vector_len = vector_length_encoding(this);
23320 BasicType etype = this->bottom_type()->is_vect()->element_basic_type();
23321 __ vprotate_imm(opcode, etype, $dst$$XMMRegister, $src$$XMMRegister, $shift$$constant, vector_len);
23322 %}
23323 ins_pipe( pipe_slow );
23324 %}
23325
23326 instruct vprorate(vec dst, vec src, vec shift) %{
23327 match(Set dst (RotateLeftV src shift));
23328 match(Set dst (RotateRightV src shift));
23329 format %{ "vprotate $dst,$src,$shift\t! vector rotate" %}
23330 ins_encode %{
23331 int opcode = this->ideal_Opcode();
23332 int vector_len = vector_length_encoding(this);
23333 BasicType etype = this->bottom_type()->is_vect()->element_basic_type();
23334 __ vprotate_var(opcode, etype, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
23335 %}
23336 ins_pipe( pipe_slow );
23337 %}
23338
23339 // ---------------------------------- Masked Operations ------------------------------------
23340 instruct vmasked_load_avx_non_subword(vec dst, memory mem, vec mask) %{
23341 predicate(!n->in(3)->bottom_type()->isa_pvectmask());
23342 match(Set dst (LoadVectorMasked mem mask));
23343 format %{ "vector_masked_load $dst, $mem, $mask \t! vector masked copy" %}
23344 ins_encode %{
23345 BasicType elmType = this->bottom_type()->is_vect()->element_basic_type();
23346 int vlen_enc = vector_length_encoding(this);
23347 __ vmovmask(elmType, $dst$$XMMRegister, $mem$$Address, $mask$$XMMRegister, vlen_enc);
23348 %}
23349 ins_pipe( pipe_slow );
23350 %}
23351
23352
23353 instruct vmasked_load_evex(vec dst, memory mem, kReg mask) %{
23354 predicate(n->in(3)->bottom_type()->isa_pvectmask());
23355 match(Set dst (LoadVectorMasked mem mask));
23356 format %{ "vector_masked_load $dst, $mem, $mask \t! vector masked copy" %}
23357 ins_encode %{
23358 BasicType elmType = this->bottom_type()->is_vect()->element_basic_type();
23359 int vector_len = vector_length_encoding(this);
23360 __ evmovdqu(elmType, $mask$$KRegister, $dst$$XMMRegister, $mem$$Address, false, vector_len);
23361 %}
23362 ins_pipe( pipe_slow );
23363 %}
23364
23365 instruct vmasked_store_avx_non_subword(memory mem, vec src, vec mask) %{
23366 predicate(!n->in(3)->in(2)->bottom_type()->isa_pvectmask());
23367 match(Set mem (StoreVectorMasked mem (Binary src mask)));
23368 format %{ "vector_masked_store $mem, $src, $mask \t! vector masked store" %}
23369 ins_encode %{
23370 const MachNode* src_node = static_cast<const MachNode*>(this->in(this->operand_index($src)));
23371 int vlen_enc = vector_length_encoding(src_node);
23372 BasicType elmType = src_node->bottom_type()->is_vect()->element_basic_type();
23373 __ vmovmask(elmType, $mem$$Address, $src$$XMMRegister, $mask$$XMMRegister, vlen_enc);
23374 %}
23375 ins_pipe( pipe_slow );
23376 %}
23377
23378 instruct vmasked_store_evex(memory mem, vec src, kReg mask) %{
23379 predicate(n->in(3)->in(2)->bottom_type()->isa_pvectmask());
23380 match(Set mem (StoreVectorMasked mem (Binary src mask)));
23381 format %{ "vector_masked_store $mem, $src, $mask \t! vector masked store" %}
23382 ins_encode %{
23383 const MachNode* src_node = static_cast<const MachNode*>(this->in(this->operand_index($src)));
23384 BasicType elmType = src_node->bottom_type()->is_vect()->element_basic_type();
23385 int vlen_enc = vector_length_encoding(src_node);
23386 __ evmovdqu(elmType, $mask$$KRegister, $mem$$Address, $src$$XMMRegister, true, vlen_enc);
23387 %}
23388 ins_pipe( pipe_slow );
23389 %}
23390
23391 instruct verify_vector_alignment(rRegP addr, immL32 mask, rFlagsReg cr) %{
23392 match(Set addr (VerifyVectorAlignment addr mask));
23393 effect(KILL cr);
23394 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
23395 ins_encode %{
23396 Label Lskip;
23397 // check if masked bits of addr are zero
23398 __ testq($addr$$Register, $mask$$constant);
23399 __ jccb(Assembler::equal, Lskip);
23400 __ stop("verify_vector_alignment found a misaligned vector memory access");
23401 __ bind(Lskip);
23402 %}
23403 ins_pipe(pipe_slow);
23404 %}
23405
23406 instruct vmask_cmp_node(rRegI dst, vec src1, vec src2, kReg mask, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
23407 match(Set dst (VectorCmpMasked src1 (Binary src2 mask)));
23408 effect(TEMP_DEF dst, TEMP ktmp1, TEMP ktmp2, KILL cr);
23409 format %{ "vector_mask_cmp $src1, $src2, $mask \t! vector mask comparison" %}
23410 ins_encode %{
23411 assert(vector_length_encoding(this, $src1) == vector_length_encoding(this, $src2), "mismatch");
23412 assert(Matcher::vector_element_basic_type(this, $src1) == Matcher::vector_element_basic_type(this, $src2), "mismatch");
23413
23414 Label DONE;
23415 int vlen_enc = vector_length_encoding(this, $src1);
23416 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src1);
23417
23418 __ knotql($ktmp2$$KRegister, $mask$$KRegister);
23419 __ mov64($dst$$Register, -1L);
23420 __ evpcmp(elem_bt, $ktmp1$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, Assembler::eq, vlen_enc);
23421 __ kortestql($ktmp2$$KRegister, $ktmp1$$KRegister);
23422 __ jccb(Assembler::carrySet, DONE);
23423 __ kmovql($dst$$Register, $ktmp1$$KRegister);
23424 __ notq($dst$$Register);
23425 __ tzcntq($dst$$Register, $dst$$Register);
23426 __ bind(DONE);
23427 %}
23428 ins_pipe( pipe_slow );
23429 %}
23430
23431
23432 instruct vmask_gen(kReg dst, rRegL len, rRegL temp, rFlagsReg cr) %{
23433 match(Set dst (VectorMaskGen len));
23434 effect(TEMP temp, KILL cr);
23435 format %{ "vector_mask_gen32 $dst, $len \t! vector mask generator" %}
23436 ins_encode %{
23437 __ genmask($dst$$KRegister, $len$$Register, $temp$$Register);
23438 %}
23439 ins_pipe( pipe_slow );
23440 %}
23441
23442 instruct vmask_gen_imm(kReg dst, immL len, rRegL temp) %{
23443 match(Set dst (VectorMaskGen len));
23444 format %{ "vector_mask_gen $len \t! vector mask generator" %}
23445 effect(TEMP temp);
23446 ins_encode %{
23447 if ($len$$constant > 0) {
23448 __ mov64($temp$$Register, right_n_bits($len$$constant));
23449 __ kmovql($dst$$KRegister, $temp$$Register);
23450 } else {
23451 __ kxorql($dst$$KRegister, $dst$$KRegister, $dst$$KRegister);
23452 }
23453 %}
23454 ins_pipe( pipe_slow );
23455 %}
23456
23457 instruct vmask_tolong_evex(rRegL dst, kReg mask, rFlagsReg cr) %{
23458 predicate(n->in(1)->bottom_type()->isa_pvectmask());
23459 match(Set dst (VectorMaskToLong mask));
23460 effect(TEMP dst, KILL cr);
23461 format %{ "vector_tolong_evex $dst, $mask \t! vector mask tolong" %}
23462 ins_encode %{
23463 int opcode = this->ideal_Opcode();
23464 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23465 int mask_len = Matcher::vector_length(this, $mask);
23466 int mask_size = mask_len * type2aelembytes(mbt);
23467 int vlen_enc = vector_length_encoding(this, $mask);
23468 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
23469 $dst$$Register, mask_len, mask_size, vlen_enc);
23470 %}
23471 ins_pipe( pipe_slow );
23472 %}
23473
23474 instruct vmask_tolong_bool(rRegL dst, vec mask, vec xtmp, rFlagsReg cr) %{
23475 predicate(n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23476 match(Set dst (VectorMaskToLong mask));
23477 format %{ "vector_tolong_bool $dst, $mask \t! using $xtmp as TEMP" %}
23478 effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
23479 ins_encode %{
23480 int opcode = this->ideal_Opcode();
23481 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23482 int mask_len = Matcher::vector_length(this, $mask);
23483 int vlen_enc = vector_length_encoding(this, $mask);
23484 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23485 $dst$$Register, mask_len, mbt, vlen_enc);
23486 %}
23487 ins_pipe( pipe_slow );
23488 %}
23489
23490 instruct vmask_tolong_avx(rRegL dst, vec mask, immI size, vec xtmp, rFlagsReg cr) %{
23491 predicate(n->in(1)->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23492 match(Set dst (VectorMaskToLong (VectorStoreMask mask size)));
23493 format %{ "vector_tolong_avx $dst, $mask \t! using $xtmp as TEMP" %}
23494 effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
23495 ins_encode %{
23496 int opcode = this->ideal_Opcode();
23497 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23498 int mask_len = Matcher::vector_length(this, $mask);
23499 int vlen_enc = vector_length_encoding(this, $mask);
23500 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23501 $dst$$Register, mask_len, mbt, vlen_enc);
23502 %}
23503 ins_pipe( pipe_slow );
23504 %}
23505
23506 instruct vmask_truecount_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
23507 predicate(n->in(1)->bottom_type()->isa_pvectmask());
23508 match(Set dst (VectorMaskTrueCount mask));
23509 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
23510 format %{ "vector_truecount_evex $dst, $mask \t! using $tmp as TEMP" %}
23511 ins_encode %{
23512 int opcode = this->ideal_Opcode();
23513 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23514 int mask_len = Matcher::vector_length(this, $mask);
23515 int mask_size = mask_len * type2aelembytes(mbt);
23516 int vlen_enc = vector_length_encoding(this, $mask);
23517 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
23518 $tmp$$Register, mask_len, mask_size, vlen_enc);
23519 %}
23520 ins_pipe( pipe_slow );
23521 %}
23522
23523 instruct vmask_truecount_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23524 predicate(n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23525 match(Set dst (VectorMaskTrueCount mask));
23526 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23527 format %{ "vector_truecount_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23528 ins_encode %{
23529 int opcode = this->ideal_Opcode();
23530 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23531 int mask_len = Matcher::vector_length(this, $mask);
23532 int vlen_enc = vector_length_encoding(this, $mask);
23533 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23534 $tmp$$Register, mask_len, mbt, vlen_enc);
23535 %}
23536 ins_pipe( pipe_slow );
23537 %}
23538
23539 instruct vmask_truecount_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23540 predicate(n->in(1)->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23541 match(Set dst (VectorMaskTrueCount (VectorStoreMask mask size)));
23542 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23543 format %{ "vector_truecount_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23544 ins_encode %{
23545 int opcode = this->ideal_Opcode();
23546 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23547 int mask_len = Matcher::vector_length(this, $mask);
23548 int vlen_enc = vector_length_encoding(this, $mask);
23549 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23550 $tmp$$Register, mask_len, mbt, vlen_enc);
23551 %}
23552 ins_pipe( pipe_slow );
23553 %}
23554
23555 instruct vmask_first_or_last_true_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
23556 predicate(n->in(1)->bottom_type()->isa_pvectmask());
23557 match(Set dst (VectorMaskFirstTrue mask));
23558 match(Set dst (VectorMaskLastTrue mask));
23559 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
23560 format %{ "vector_mask_first_or_last_true_evex $dst, $mask \t! using $tmp as TEMP" %}
23561 ins_encode %{
23562 int opcode = this->ideal_Opcode();
23563 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23564 int mask_len = Matcher::vector_length(this, $mask);
23565 int mask_size = mask_len * type2aelembytes(mbt);
23566 int vlen_enc = vector_length_encoding(this, $mask);
23567 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
23568 $tmp$$Register, mask_len, mask_size, vlen_enc);
23569 %}
23570 ins_pipe( pipe_slow );
23571 %}
23572
23573 instruct vmask_first_or_last_true_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23574 predicate(n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23575 match(Set dst (VectorMaskFirstTrue mask));
23576 match(Set dst (VectorMaskLastTrue mask));
23577 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23578 format %{ "vector_mask_first_or_last_true_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23579 ins_encode %{
23580 int opcode = this->ideal_Opcode();
23581 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23582 int mask_len = Matcher::vector_length(this, $mask);
23583 int vlen_enc = vector_length_encoding(this, $mask);
23584 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23585 $tmp$$Register, mask_len, mbt, vlen_enc);
23586 %}
23587 ins_pipe( pipe_slow );
23588 %}
23589
23590 instruct vmask_first_or_last_true_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23591 predicate(n->in(1)->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23592 match(Set dst (VectorMaskFirstTrue (VectorStoreMask mask size)));
23593 match(Set dst (VectorMaskLastTrue (VectorStoreMask mask size)));
23594 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23595 format %{ "vector_mask_first_or_last_true_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23596 ins_encode %{
23597 int opcode = this->ideal_Opcode();
23598 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23599 int mask_len = Matcher::vector_length(this, $mask);
23600 int vlen_enc = vector_length_encoding(this, $mask);
23601 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23602 $tmp$$Register, mask_len, mbt, vlen_enc);
23603 %}
23604 ins_pipe( pipe_slow );
23605 %}
23606
23607 // --------------------------------- Compress/Expand Operations ---------------------------
23608 instruct vcompress_reg_avx(vec dst, vec src, vec mask, rRegI rtmp, rRegL rscratch, vec perm, vec xtmp, rFlagsReg cr) %{
23609 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n) <= 32);
23610 match(Set dst (CompressV src mask));
23611 match(Set dst (ExpandV src mask));
23612 effect(TEMP_DEF dst, TEMP perm, TEMP xtmp, TEMP rtmp, TEMP rscratch, KILL cr);
23613 format %{ "vector_compress $dst, $src, $mask \t!using $xtmp, $rtmp, $rscratch and $perm as TEMP" %}
23614 ins_encode %{
23615 int opcode = this->ideal_Opcode();
23616 int vlen_enc = vector_length_encoding(this);
23617 BasicType bt = Matcher::vector_element_basic_type(this);
23618 __ vector_compress_expand_avx2(opcode, $dst$$XMMRegister, $src$$XMMRegister, $mask$$XMMRegister, $rtmp$$Register,
23619 $rscratch$$Register, $perm$$XMMRegister, $xtmp$$XMMRegister, bt, vlen_enc);
23620 %}
23621 ins_pipe( pipe_slow );
23622 %}
23623
23624 instruct vcompress_expand_reg_evex(vec dst, vec src, kReg mask) %{
23625 predicate(VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64);
23626 match(Set dst (CompressV src mask));
23627 match(Set dst (ExpandV src mask));
23628 format %{ "vector_compress_expand $dst, $src, $mask" %}
23629 ins_encode %{
23630 int opcode = this->ideal_Opcode();
23631 int vector_len = vector_length_encoding(this);
23632 BasicType bt = Matcher::vector_element_basic_type(this);
23633 __ vector_compress_expand(opcode, $dst$$XMMRegister, $src$$XMMRegister, $mask$$KRegister, false, bt, vector_len);
23634 %}
23635 ins_pipe( pipe_slow );
23636 %}
23637
23638 instruct vcompress_mask_reg_evex(kReg dst, kReg mask, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
23639 match(Set dst (CompressM mask));
23640 effect(TEMP rtmp1, TEMP rtmp2, KILL cr);
23641 format %{ "mask_compress_evex $dst, $mask\t! using $rtmp1 and $rtmp2 as TEMP" %}
23642 ins_encode %{
23643 assert(this->in(1)->bottom_type()->isa_pvectmask(), "");
23644 int mask_len = Matcher::vector_length(this);
23645 __ vector_mask_compress($dst$$KRegister, $mask$$KRegister, $rtmp1$$Register, $rtmp2$$Register, mask_len);
23646 %}
23647 ins_pipe( pipe_slow );
23648 %}
23649
23650 // -------------------------------- Bit and Byte Reversal Vector Operations ------------------------
23651
23652 instruct vreverse_reg(vec dst, vec src, vec xtmp1, vec xtmp2, rRegI rtmp) %{
23653 predicate(!VM_Version::supports_gfni());
23654 match(Set dst (ReverseV src));
23655 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp);
23656 format %{ "vector_reverse_bit_evex $dst, $src!\t using $xtmp1, $xtmp2 and $rtmp as TEMP" %}
23657 ins_encode %{
23658 int vec_enc = vector_length_encoding(this);
23659 BasicType bt = Matcher::vector_element_basic_type(this);
23660 __ vector_reverse_bit(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23661 $xtmp2$$XMMRegister, $rtmp$$Register, vec_enc);
23662 %}
23663 ins_pipe( pipe_slow );
23664 %}
23665
23666 instruct vreverse_reg_gfni(vec dst, vec src, vec xtmp) %{
23667 predicate(VM_Version::supports_gfni());
23668 match(Set dst (ReverseV src));
23669 effect(TEMP dst, TEMP xtmp);
23670 format %{ "vector_reverse_bit_gfni $dst, $src!\t using $xtmp as TEMP" %}
23671 ins_encode %{
23672 int vec_enc = vector_length_encoding(this);
23673 BasicType bt = Matcher::vector_element_basic_type(this);
23674 InternalAddress addr = $constantaddress(jlong(0x8040201008040201));
23675 __ vector_reverse_bit_gfni(bt, $dst$$XMMRegister, $src$$XMMRegister, addr, vec_enc,
23676 $xtmp$$XMMRegister);
23677 %}
23678 ins_pipe( pipe_slow );
23679 %}
23680
23681 instruct vreverse_byte_reg(vec dst, vec src) %{
23682 predicate(VM_Version::supports_avx512bw() || Matcher::vector_length_in_bytes(n) < 64);
23683 match(Set dst (ReverseBytesV src));
23684 effect(TEMP dst);
23685 format %{ "vector_reverse_byte $dst, $src" %}
23686 ins_encode %{
23687 int vec_enc = vector_length_encoding(this);
23688 BasicType bt = Matcher::vector_element_basic_type(this);
23689 __ vector_reverse_byte(bt, $dst$$XMMRegister, $src$$XMMRegister, vec_enc);
23690 %}
23691 ins_pipe( pipe_slow );
23692 %}
23693
23694 instruct vreverse_byte64_reg(vec dst, vec src, vec xtmp1, vec xtmp2, rRegI rtmp) %{
23695 predicate(!VM_Version::supports_avx512bw() && Matcher::vector_length_in_bytes(n) == 64);
23696 match(Set dst (ReverseBytesV src));
23697 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp);
23698 format %{ "vector_reverse_byte $dst, $src!\t using $xtmp1, $xtmp2 and $rtmp as TEMP" %}
23699 ins_encode %{
23700 int vec_enc = vector_length_encoding(this);
23701 BasicType bt = Matcher::vector_element_basic_type(this);
23702 __ vector_reverse_byte64(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23703 $xtmp2$$XMMRegister, $rtmp$$Register, vec_enc);
23704 %}
23705 ins_pipe( pipe_slow );
23706 %}
23707
23708 // ---------------------------------- Vector Count Leading Zeros -----------------------------------
23709
23710 instruct vcount_leading_zeros_IL_reg_evex(vec dst, vec src) %{
23711 predicate(is_clz_non_subword_predicate_evex(Matcher::vector_element_basic_type(n->in(1)),
23712 Matcher::vector_length_in_bytes(n->in(1))));
23713 match(Set dst (CountLeadingZerosV src));
23714 format %{ "vector_count_leading_zeros $dst, $src" %}
23715 ins_encode %{
23716 int vlen_enc = vector_length_encoding(this, $src);
23717 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23718 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, xnoreg,
23719 xnoreg, xnoreg, k0, noreg, true, vlen_enc);
23720 %}
23721 ins_pipe( pipe_slow );
23722 %}
23723
23724 instruct vcount_leading_zeros_IL_reg_evex_masked(vec dst, vec src, kReg mask) %{
23725 predicate(is_clz_non_subword_predicate_evex(Matcher::vector_element_basic_type(n->in(1)),
23726 Matcher::vector_length_in_bytes(n->in(1))));
23727 match(Set dst (CountLeadingZerosV src mask));
23728 format %{ "vector_count_leading_zeros $dst, $src, $mask" %}
23729 ins_encode %{
23730 int vlen_enc = vector_length_encoding(this, $src);
23731 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23732 __ evmovdquq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
23733 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, xnoreg, xnoreg,
23734 xnoreg, $mask$$KRegister, noreg, true, vlen_enc);
23735 %}
23736 ins_pipe( pipe_slow );
23737 %}
23738
23739 instruct vcount_leading_zeros_short_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2) %{
23740 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_SHORT &&
23741 VM_Version::supports_avx512cd() &&
23742 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64));
23743 match(Set dst (CountLeadingZerosV src));
23744 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
23745 format %{ "vector_count_leading_zeros $dst, $src!\t using $xtmp1 and $xtmp2 as TEMP" %}
23746 ins_encode %{
23747 int vlen_enc = vector_length_encoding(this, $src);
23748 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23749 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23750 $xtmp2$$XMMRegister, xnoreg, k0, noreg, true, vlen_enc);
23751 %}
23752 ins_pipe( pipe_slow );
23753 %}
23754
23755 instruct vcount_leading_zeros_byte_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, kReg ktmp, rRegP rtmp) %{
23756 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_BYTE && VM_Version::supports_avx512vlbw());
23757 match(Set dst (CountLeadingZerosV src));
23758 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP ktmp, TEMP rtmp);
23759 format %{ "vector_count_leading_zeros $dst, $src!\t using $xtmp1, $xtmp2, $xtmp3, $ktmp and $rtmp as TEMP" %}
23760 ins_encode %{
23761 int vlen_enc = vector_length_encoding(this, $src);
23762 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23763 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23764 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $ktmp$$KRegister,
23765 $rtmp$$Register, true, vlen_enc);
23766 %}
23767 ins_pipe( pipe_slow );
23768 %}
23769
23770 instruct vcount_leading_zeros_int_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3) %{
23771 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_INT &&
23772 !VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n->in(1)) < 64);
23773 match(Set dst (CountLeadingZerosV src));
23774 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3);
23775 format %{ "vector_count_leading_zeros $dst, $src\t! using $xtmp1, $xtmp2 and $xtmp3 as TEMP" %}
23776 ins_encode %{
23777 int vlen_enc = vector_length_encoding(this, $src);
23778 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23779 __ vector_count_leading_zeros_avx(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23780 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, noreg, vlen_enc);
23781 %}
23782 ins_pipe( pipe_slow );
23783 %}
23784
23785 instruct vcount_leading_zeros_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, rRegP rtmp) %{
23786 predicate(Matcher::vector_element_basic_type(n->in(1)) != T_INT &&
23787 !VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n->in(1)) < 64);
23788 match(Set dst (CountLeadingZerosV src));
23789 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp);
23790 format %{ "vector_count_leading_zeros $dst, $src\t! using $xtmp1, $xtmp2, $xtmp3, and $rtmp as TEMP" %}
23791 ins_encode %{
23792 int vlen_enc = vector_length_encoding(this, $src);
23793 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23794 __ vector_count_leading_zeros_avx(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23795 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, vlen_enc);
23796 %}
23797 ins_pipe( pipe_slow );
23798 %}
23799
23800 // ---------------------------------- Vector Masked Operations ------------------------------------
23801
23802 instruct vadd_reg_masked(vec dst, vec src2, kReg mask) %{
23803 match(Set dst (AddVB (Binary dst src2) mask));
23804 match(Set dst (AddVS (Binary dst src2) mask));
23805 match(Set dst (AddVI (Binary dst src2) mask));
23806 match(Set dst (AddVL (Binary dst src2) mask));
23807 match(Set dst (AddVF (Binary dst src2) mask));
23808 match(Set dst (AddVD (Binary dst src2) mask));
23809 format %{ "vpadd_masked $dst, $dst, $src2, $mask\t! add masked operation" %}
23810 ins_encode %{
23811 int vlen_enc = vector_length_encoding(this);
23812 BasicType bt = Matcher::vector_element_basic_type(this);
23813 int opc = this->ideal_Opcode();
23814 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23815 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
23816 %}
23817 ins_pipe( pipe_slow );
23818 %}
23819
23820 instruct vadd_mem_masked(vec dst, memory src2, kReg mask) %{
23821 match(Set dst (AddVB (Binary dst (LoadVector src2)) mask));
23822 match(Set dst (AddVS (Binary dst (LoadVector src2)) mask));
23823 match(Set dst (AddVI (Binary dst (LoadVector src2)) mask));
23824 match(Set dst (AddVL (Binary dst (LoadVector src2)) mask));
23825 match(Set dst (AddVF (Binary dst (LoadVector src2)) mask));
23826 match(Set dst (AddVD (Binary dst (LoadVector src2)) mask));
23827 format %{ "vpadd_masked $dst, $dst, $src2, $mask\t! add masked operation" %}
23828 ins_encode %{
23829 int vlen_enc = vector_length_encoding(this);
23830 BasicType bt = Matcher::vector_element_basic_type(this);
23831 int opc = this->ideal_Opcode();
23832 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23833 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
23834 %}
23835 ins_pipe( pipe_slow );
23836 %}
23837
23838 instruct vxor_reg_masked(vec dst, vec src2, kReg mask) %{
23839 match(Set dst (XorV (Binary dst src2) mask));
23840 format %{ "vxor_masked $dst, $dst, $src2, $mask\t! xor masked operation" %}
23841 ins_encode %{
23842 int vlen_enc = vector_length_encoding(this);
23843 BasicType bt = Matcher::vector_element_basic_type(this);
23844 int opc = this->ideal_Opcode();
23845 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23846 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
23847 %}
23848 ins_pipe( pipe_slow );
23849 %}
23850
23851 instruct vxor_mem_masked(vec dst, memory src2, kReg mask) %{
23852 match(Set dst (XorV (Binary dst (LoadVector src2)) mask));
23853 format %{ "vxor_masked $dst, $dst, $src2, $mask\t! xor masked operation" %}
23854 ins_encode %{
23855 int vlen_enc = vector_length_encoding(this);
23856 BasicType bt = Matcher::vector_element_basic_type(this);
23857 int opc = this->ideal_Opcode();
23858 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23859 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
23860 %}
23861 ins_pipe( pipe_slow );
23862 %}
23863
23864 instruct vor_reg_masked(vec dst, vec src2, kReg mask) %{
23865 match(Set dst (OrV (Binary dst src2) mask));
23866 format %{ "vor_masked $dst, $dst, $src2, $mask\t! or masked operation" %}
23867 ins_encode %{
23868 int vlen_enc = vector_length_encoding(this);
23869 BasicType bt = Matcher::vector_element_basic_type(this);
23870 int opc = this->ideal_Opcode();
23871 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23872 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
23873 %}
23874 ins_pipe( pipe_slow );
23875 %}
23876
23877 instruct vor_mem_masked(vec dst, memory src2, kReg mask) %{
23878 match(Set dst (OrV (Binary dst (LoadVector src2)) mask));
23879 format %{ "vor_masked $dst, $dst, $src2, $mask\t! or masked operation" %}
23880 ins_encode %{
23881 int vlen_enc = vector_length_encoding(this);
23882 BasicType bt = Matcher::vector_element_basic_type(this);
23883 int opc = this->ideal_Opcode();
23884 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23885 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
23886 %}
23887 ins_pipe( pipe_slow );
23888 %}
23889
23890 instruct vand_reg_masked(vec dst, vec src2, kReg mask) %{
23891 match(Set dst (AndV (Binary dst src2) mask));
23892 format %{ "vand_masked $dst, $dst, $src2, $mask\t! and masked operation" %}
23893 ins_encode %{
23894 int vlen_enc = vector_length_encoding(this);
23895 BasicType bt = Matcher::vector_element_basic_type(this);
23896 int opc = this->ideal_Opcode();
23897 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23898 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
23899 %}
23900 ins_pipe( pipe_slow );
23901 %}
23902
23903 instruct vand_mem_masked(vec dst, memory src2, kReg mask) %{
23904 match(Set dst (AndV (Binary dst (LoadVector src2)) mask));
23905 format %{ "vand_masked $dst, $dst, $src2, $mask\t! and masked operation" %}
23906 ins_encode %{
23907 int vlen_enc = vector_length_encoding(this);
23908 BasicType bt = Matcher::vector_element_basic_type(this);
23909 int opc = this->ideal_Opcode();
23910 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23911 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
23912 %}
23913 ins_pipe( pipe_slow );
23914 %}
23915
23916 instruct vsub_reg_masked(vec dst, vec src2, kReg mask) %{
23917 match(Set dst (SubVB (Binary dst src2) mask));
23918 match(Set dst (SubVS (Binary dst src2) mask));
23919 match(Set dst (SubVI (Binary dst src2) mask));
23920 match(Set dst (SubVL (Binary dst src2) mask));
23921 match(Set dst (SubVF (Binary dst src2) mask));
23922 match(Set dst (SubVD (Binary dst src2) mask));
23923 format %{ "vpsub_masked $dst, $dst, $src2, $mask\t! sub masked operation" %}
23924 ins_encode %{
23925 int vlen_enc = vector_length_encoding(this);
23926 BasicType bt = Matcher::vector_element_basic_type(this);
23927 int opc = this->ideal_Opcode();
23928 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23929 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
23930 %}
23931 ins_pipe( pipe_slow );
23932 %}
23933
23934 instruct vsub_mem_masked(vec dst, memory src2, kReg mask) %{
23935 match(Set dst (SubVB (Binary dst (LoadVector src2)) mask));
23936 match(Set dst (SubVS (Binary dst (LoadVector src2)) mask));
23937 match(Set dst (SubVI (Binary dst (LoadVector src2)) mask));
23938 match(Set dst (SubVL (Binary dst (LoadVector src2)) mask));
23939 match(Set dst (SubVF (Binary dst (LoadVector src2)) mask));
23940 match(Set dst (SubVD (Binary dst (LoadVector src2)) mask));
23941 format %{ "vpsub_masked $dst, $dst, $src2, $mask\t! sub masked operation" %}
23942 ins_encode %{
23943 int vlen_enc = vector_length_encoding(this);
23944 BasicType bt = Matcher::vector_element_basic_type(this);
23945 int opc = this->ideal_Opcode();
23946 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23947 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
23948 %}
23949 ins_pipe( pipe_slow );
23950 %}
23951
23952 instruct vmul_reg_masked(vec dst, vec src2, kReg mask) %{
23953 match(Set dst (MulVS (Binary dst src2) mask));
23954 match(Set dst (MulVI (Binary dst src2) mask));
23955 match(Set dst (MulVL (Binary dst src2) mask));
23956 match(Set dst (MulVF (Binary dst src2) mask));
23957 match(Set dst (MulVD (Binary dst src2) mask));
23958 format %{ "vpmul_masked $dst, $dst, $src2, $mask\t! mul masked operation" %}
23959 ins_encode %{
23960 int vlen_enc = vector_length_encoding(this);
23961 BasicType bt = Matcher::vector_element_basic_type(this);
23962 int opc = this->ideal_Opcode();
23963 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23964 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
23965 %}
23966 ins_pipe( pipe_slow );
23967 %}
23968
23969 instruct vmul_mem_masked(vec dst, memory src2, kReg mask) %{
23970 match(Set dst (MulVS (Binary dst (LoadVector src2)) mask));
23971 match(Set dst (MulVI (Binary dst (LoadVector src2)) mask));
23972 match(Set dst (MulVL (Binary dst (LoadVector src2)) mask));
23973 match(Set dst (MulVF (Binary dst (LoadVector src2)) mask));
23974 match(Set dst (MulVD (Binary dst (LoadVector src2)) mask));
23975 format %{ "vpmul_masked $dst, $dst, $src2, $mask\t! mul masked operation" %}
23976 ins_encode %{
23977 int vlen_enc = vector_length_encoding(this);
23978 BasicType bt = Matcher::vector_element_basic_type(this);
23979 int opc = this->ideal_Opcode();
23980 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23981 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
23982 %}
23983 ins_pipe( pipe_slow );
23984 %}
23985
23986 instruct vsqrt_reg_masked(vec dst, kReg mask) %{
23987 match(Set dst (SqrtVF dst mask));
23988 match(Set dst (SqrtVD dst mask));
23989 format %{ "vpsqrt_masked $dst, $mask\t! sqrt masked operation" %}
23990 ins_encode %{
23991 int vlen_enc = vector_length_encoding(this);
23992 BasicType bt = Matcher::vector_element_basic_type(this);
23993 int opc = this->ideal_Opcode();
23994 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23995 $dst$$XMMRegister, $dst$$XMMRegister, true, vlen_enc);
23996 %}
23997 ins_pipe( pipe_slow );
23998 %}
23999
24000 instruct vdiv_reg_masked(vec dst, vec src2, kReg mask) %{
24001 match(Set dst (DivVF (Binary dst src2) mask));
24002 match(Set dst (DivVD (Binary dst src2) mask));
24003 format %{ "vpdiv_masked $dst, $dst, $src2, $mask\t! div masked operation" %}
24004 ins_encode %{
24005 int vlen_enc = vector_length_encoding(this);
24006 BasicType bt = Matcher::vector_element_basic_type(this);
24007 int opc = this->ideal_Opcode();
24008 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24009 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24010 %}
24011 ins_pipe( pipe_slow );
24012 %}
24013
24014 instruct vdiv_mem_masked(vec dst, memory src2, kReg mask) %{
24015 match(Set dst (DivVF (Binary dst (LoadVector src2)) mask));
24016 match(Set dst (DivVD (Binary dst (LoadVector src2)) mask));
24017 format %{ "vpdiv_masked $dst, $dst, $src2, $mask\t! div masked operation" %}
24018 ins_encode %{
24019 int vlen_enc = vector_length_encoding(this);
24020 BasicType bt = Matcher::vector_element_basic_type(this);
24021 int opc = this->ideal_Opcode();
24022 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24023 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24024 %}
24025 ins_pipe( pipe_slow );
24026 %}
24027
24028
24029 instruct vrol_imm_masked(vec dst, immI8 shift, kReg mask) %{
24030 match(Set dst (RotateLeftV (Binary dst shift) mask));
24031 match(Set dst (RotateRightV (Binary dst shift) mask));
24032 format %{ "vprotate_imm_masked $dst, $dst, $shift, $mask\t! rotate masked operation" %}
24033 ins_encode %{
24034 int vlen_enc = vector_length_encoding(this);
24035 BasicType bt = Matcher::vector_element_basic_type(this);
24036 int opc = this->ideal_Opcode();
24037 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24038 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24039 %}
24040 ins_pipe( pipe_slow );
24041 %}
24042
24043 instruct vrol_reg_masked(vec dst, vec src2, kReg mask) %{
24044 match(Set dst (RotateLeftV (Binary dst src2) mask));
24045 match(Set dst (RotateRightV (Binary dst src2) mask));
24046 format %{ "vrotate_masked $dst, $dst, $src2, $mask\t! rotate masked operation" %}
24047 ins_encode %{
24048 int vlen_enc = vector_length_encoding(this);
24049 BasicType bt = Matcher::vector_element_basic_type(this);
24050 int opc = this->ideal_Opcode();
24051 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24052 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24053 %}
24054 ins_pipe( pipe_slow );
24055 %}
24056
24057 instruct vlshift_imm_masked(vec dst, immI8 shift, kReg mask) %{
24058 match(Set dst (LShiftVS (Binary dst (LShiftCntV shift)) mask));
24059 match(Set dst (LShiftVI (Binary dst (LShiftCntV shift)) mask));
24060 match(Set dst (LShiftVL (Binary dst (LShiftCntV shift)) mask));
24061 format %{ "vplshift_imm_masked $dst, $dst, $shift, $mask\t! lshift masked operation" %}
24062 ins_encode %{
24063 int vlen_enc = vector_length_encoding(this);
24064 BasicType bt = Matcher::vector_element_basic_type(this);
24065 int opc = this->ideal_Opcode();
24066 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24067 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24068 %}
24069 ins_pipe( pipe_slow );
24070 %}
24071
24072 instruct vlshift_reg_masked(vec dst, vec src2, kReg mask) %{
24073 predicate(!n->as_ShiftV()->is_var_shift());
24074 match(Set dst (LShiftVS (Binary dst src2) mask));
24075 match(Set dst (LShiftVI (Binary dst src2) mask));
24076 match(Set dst (LShiftVL (Binary dst src2) mask));
24077 format %{ "vplshift_masked $dst, $dst, $src2, $mask\t! lshift masked operation" %}
24078 ins_encode %{
24079 int vlen_enc = vector_length_encoding(this);
24080 BasicType bt = Matcher::vector_element_basic_type(this);
24081 int opc = this->ideal_Opcode();
24082 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24083 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, false);
24084 %}
24085 ins_pipe( pipe_slow );
24086 %}
24087
24088 instruct vlshiftv_reg_masked(vec dst, vec src2, kReg mask) %{
24089 predicate(n->as_ShiftV()->is_var_shift());
24090 match(Set dst (LShiftVS (Binary dst src2) mask));
24091 match(Set dst (LShiftVI (Binary dst src2) mask));
24092 match(Set dst (LShiftVL (Binary dst src2) mask));
24093 format %{ "vplshiftv_masked $dst, $dst, $src2, $mask\t! lshift masked operation" %}
24094 ins_encode %{
24095 int vlen_enc = vector_length_encoding(this);
24096 BasicType bt = Matcher::vector_element_basic_type(this);
24097 int opc = this->ideal_Opcode();
24098 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24099 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, true);
24100 %}
24101 ins_pipe( pipe_slow );
24102 %}
24103
24104 instruct vrshift_imm_masked(vec dst, immI8 shift, kReg mask) %{
24105 match(Set dst (RShiftVS (Binary dst (RShiftCntV shift)) mask));
24106 match(Set dst (RShiftVI (Binary dst (RShiftCntV shift)) mask));
24107 match(Set dst (RShiftVL (Binary dst (RShiftCntV shift)) mask));
24108 format %{ "vprshift_imm_masked $dst, $dst, $shift, $mask\t! rshift masked operation" %}
24109 ins_encode %{
24110 int vlen_enc = vector_length_encoding(this);
24111 BasicType bt = Matcher::vector_element_basic_type(this);
24112 int opc = this->ideal_Opcode();
24113 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24114 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24115 %}
24116 ins_pipe( pipe_slow );
24117 %}
24118
24119 instruct vrshift_reg_masked(vec dst, vec src2, kReg mask) %{
24120 predicate(!n->as_ShiftV()->is_var_shift());
24121 match(Set dst (RShiftVS (Binary dst src2) mask));
24122 match(Set dst (RShiftVI (Binary dst src2) mask));
24123 match(Set dst (RShiftVL (Binary dst src2) mask));
24124 format %{ "vprshift_masked $dst, $dst, $src2, $mask\t! rshift masked operation" %}
24125 ins_encode %{
24126 int vlen_enc = vector_length_encoding(this);
24127 BasicType bt = Matcher::vector_element_basic_type(this);
24128 int opc = this->ideal_Opcode();
24129 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24130 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, false);
24131 %}
24132 ins_pipe( pipe_slow );
24133 %}
24134
24135 instruct vrshiftv_reg_masked(vec dst, vec src2, kReg mask) %{
24136 predicate(n->as_ShiftV()->is_var_shift());
24137 match(Set dst (RShiftVS (Binary dst src2) mask));
24138 match(Set dst (RShiftVI (Binary dst src2) mask));
24139 match(Set dst (RShiftVL (Binary dst src2) mask));
24140 format %{ "vprshiftv_masked $dst, $dst, $src2, $mask\t! rshift masked operation" %}
24141 ins_encode %{
24142 int vlen_enc = vector_length_encoding(this);
24143 BasicType bt = Matcher::vector_element_basic_type(this);
24144 int opc = this->ideal_Opcode();
24145 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24146 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, true);
24147 %}
24148 ins_pipe( pipe_slow );
24149 %}
24150
24151 instruct vurshift_imm_masked(vec dst, immI8 shift, kReg mask) %{
24152 match(Set dst (URShiftVS (Binary dst (RShiftCntV shift)) mask));
24153 match(Set dst (URShiftVI (Binary dst (RShiftCntV shift)) mask));
24154 match(Set dst (URShiftVL (Binary dst (RShiftCntV shift)) mask));
24155 format %{ "vpurshift_imm_masked $dst, $dst, $shift, $mask\t! urshift masked operation" %}
24156 ins_encode %{
24157 int vlen_enc = vector_length_encoding(this);
24158 BasicType bt = Matcher::vector_element_basic_type(this);
24159 int opc = this->ideal_Opcode();
24160 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24161 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24162 %}
24163 ins_pipe( pipe_slow );
24164 %}
24165
24166 instruct vurshift_reg_masked(vec dst, vec src2, kReg mask) %{
24167 predicate(!n->as_ShiftV()->is_var_shift());
24168 match(Set dst (URShiftVS (Binary dst src2) mask));
24169 match(Set dst (URShiftVI (Binary dst src2) mask));
24170 match(Set dst (URShiftVL (Binary dst src2) mask));
24171 format %{ "vpurshift_masked $dst, $dst, $src2, $mask\t! urshift masked operation" %}
24172 ins_encode %{
24173 int vlen_enc = vector_length_encoding(this);
24174 BasicType bt = Matcher::vector_element_basic_type(this);
24175 int opc = this->ideal_Opcode();
24176 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24177 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, false);
24178 %}
24179 ins_pipe( pipe_slow );
24180 %}
24181
24182 instruct vurshiftv_reg_masked(vec dst, vec src2, kReg mask) %{
24183 predicate(n->as_ShiftV()->is_var_shift());
24184 match(Set dst (URShiftVS (Binary dst src2) mask));
24185 match(Set dst (URShiftVI (Binary dst src2) mask));
24186 match(Set dst (URShiftVL (Binary dst src2) mask));
24187 format %{ "vpurshiftv_masked $dst, $dst, $src2, $mask\t! urshift masked operation" %}
24188 ins_encode %{
24189 int vlen_enc = vector_length_encoding(this);
24190 BasicType bt = Matcher::vector_element_basic_type(this);
24191 int opc = this->ideal_Opcode();
24192 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24193 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, true);
24194 %}
24195 ins_pipe( pipe_slow );
24196 %}
24197
24198 instruct vmaxv_reg_masked(vec dst, vec src2, kReg mask) %{
24199 match(Set dst (MaxV (Binary dst src2) mask));
24200 format %{ "vpmax_masked $dst, $dst, $src2, $mask\t! max masked operation" %}
24201 ins_encode %{
24202 int vlen_enc = vector_length_encoding(this);
24203 BasicType bt = Matcher::vector_element_basic_type(this);
24204 int opc = this->ideal_Opcode();
24205 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24206 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24207 %}
24208 ins_pipe( pipe_slow );
24209 %}
24210
24211 instruct vmaxv_mem_masked(vec dst, memory src2, kReg mask) %{
24212 match(Set dst (MaxV (Binary dst (LoadVector src2)) mask));
24213 format %{ "vpmax_masked $dst, $dst, $src2, $mask\t! max masked operation" %}
24214 ins_encode %{
24215 int vlen_enc = vector_length_encoding(this);
24216 BasicType bt = Matcher::vector_element_basic_type(this);
24217 int opc = this->ideal_Opcode();
24218 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24219 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24220 %}
24221 ins_pipe( pipe_slow );
24222 %}
24223
24224 instruct vminv_reg_masked(vec dst, vec src2, kReg mask) %{
24225 match(Set dst (MinV (Binary dst src2) mask));
24226 format %{ "vpmin_masked $dst, $dst, $src2, $mask\t! min masked operation" %}
24227 ins_encode %{
24228 int vlen_enc = vector_length_encoding(this);
24229 BasicType bt = Matcher::vector_element_basic_type(this);
24230 int opc = this->ideal_Opcode();
24231 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24232 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24233 %}
24234 ins_pipe( pipe_slow );
24235 %}
24236
24237 instruct vminv_mem_masked(vec dst, memory src2, kReg mask) %{
24238 match(Set dst (MinV (Binary dst (LoadVector src2)) mask));
24239 format %{ "vpmin_masked $dst, $dst, $src2, $mask\t! min masked operation" %}
24240 ins_encode %{
24241 int vlen_enc = vector_length_encoding(this);
24242 BasicType bt = Matcher::vector_element_basic_type(this);
24243 int opc = this->ideal_Opcode();
24244 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24245 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24246 %}
24247 ins_pipe( pipe_slow );
24248 %}
24249
24250 instruct vrearrangev_reg_masked(vec dst, vec src2, kReg mask) %{
24251 match(Set dst (VectorRearrange (Binary dst src2) mask));
24252 format %{ "vprearrange_masked $dst, $dst, $src2, $mask\t! rearrange masked operation" %}
24253 ins_encode %{
24254 int vlen_enc = vector_length_encoding(this);
24255 BasicType bt = Matcher::vector_element_basic_type(this);
24256 int opc = this->ideal_Opcode();
24257 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24258 $dst$$XMMRegister, $src2$$XMMRegister, false, vlen_enc);
24259 %}
24260 ins_pipe( pipe_slow );
24261 %}
24262
24263 instruct vabs_masked(vec dst, kReg mask) %{
24264 match(Set dst (AbsVB dst mask));
24265 match(Set dst (AbsVS dst mask));
24266 match(Set dst (AbsVI dst mask));
24267 match(Set dst (AbsVL dst mask));
24268 format %{ "vabs_masked $dst, $mask \t! vabs masked operation" %}
24269 ins_encode %{
24270 int vlen_enc = vector_length_encoding(this);
24271 BasicType bt = Matcher::vector_element_basic_type(this);
24272 int opc = this->ideal_Opcode();
24273 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24274 $dst$$XMMRegister, $dst$$XMMRegister, true, vlen_enc);
24275 %}
24276 ins_pipe( pipe_slow );
24277 %}
24278
24279 instruct vfma_reg_masked(vec dst, vec src2, vec src3, kReg mask) %{
24280 match(Set dst (FmaVF (Binary dst src2) (Binary src3 mask)));
24281 match(Set dst (FmaVD (Binary dst src2) (Binary src3 mask)));
24282 format %{ "vfma_masked $dst, $src2, $src3, $mask \t! vfma masked operation" %}
24283 ins_encode %{
24284 assert(UseFMA, "Needs FMA instructions support.");
24285 int vlen_enc = vector_length_encoding(this);
24286 BasicType bt = Matcher::vector_element_basic_type(this);
24287 int opc = this->ideal_Opcode();
24288 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24289 $src2$$XMMRegister, $src3$$XMMRegister, true, vlen_enc);
24290 %}
24291 ins_pipe( pipe_slow );
24292 %}
24293
24294 instruct vfma_mem_masked(vec dst, vec src2, memory src3, kReg mask) %{
24295 match(Set dst (FmaVF (Binary dst src2) (Binary (LoadVector src3) mask)));
24296 match(Set dst (FmaVD (Binary dst src2) (Binary (LoadVector src3) mask)));
24297 format %{ "vfma_masked $dst, $src2, $src3, $mask \t! vfma masked operation" %}
24298 ins_encode %{
24299 assert(UseFMA, "Needs FMA instructions support.");
24300 int vlen_enc = vector_length_encoding(this);
24301 BasicType bt = Matcher::vector_element_basic_type(this);
24302 int opc = this->ideal_Opcode();
24303 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24304 $src2$$XMMRegister, $src3$$Address, true, vlen_enc);
24305 %}
24306 ins_pipe( pipe_slow );
24307 %}
24308
24309 instruct evcmp_masked(kReg dst, vec src1, vec src2, immI8 cond, kReg mask) %{
24310 match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond mask)));
24311 format %{ "vcmp_masked $dst, $src1, $src2, $cond, $mask" %}
24312 ins_encode %{
24313 assert(bottom_type()->isa_pvectmask(), "TypePVectMask expected");
24314 int vlen_enc = vector_length_encoding(this, $src1);
24315 BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
24316
24317 // Comparison i
24318 switch (src1_elem_bt) {
24319 case T_BYTE: {
24320 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24321 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24322 __ evpcmpb($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24323 break;
24324 }
24325 case T_SHORT: {
24326 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24327 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24328 __ evpcmpw($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24329 break;
24330 }
24331 case T_INT: {
24332 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24333 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24334 __ evpcmpd($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24335 break;
24336 }
24337 case T_LONG: {
24338 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24339 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24340 __ evpcmpq($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24341 break;
24342 }
24343 case T_FLOAT: {
24344 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
24345 __ evcmpps($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
24346 break;
24347 }
24348 case T_DOUBLE: {
24349 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
24350 __ evcmppd($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
24351 break;
24352 }
24353 default: assert(false, "%s", type2name(src1_elem_bt)); break;
24354 }
24355 %}
24356 ins_pipe( pipe_slow );
24357 %}
24358
24359 instruct mask_all_evexI_LE32(kReg dst, rRegI src) %{
24360 predicate(Matcher::vector_length(n) <= 32);
24361 match(Set dst (MaskAll src));
24362 format %{ "mask_all_evexI_LE32 $dst, $src \t" %}
24363 ins_encode %{
24364 int mask_len = Matcher::vector_length(this);
24365 __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
24366 %}
24367 ins_pipe( pipe_slow );
24368 %}
24369
24370 instruct mask_not_immLT8(kReg dst, kReg src, rRegI rtmp, kReg ktmp, immI_M1 cnt) %{
24371 predicate(Matcher::vector_length(n) < 8 && VM_Version::supports_avx512dq());
24372 match(Set dst (XorVMask src (MaskAll cnt)));
24373 effect(TEMP_DEF dst, TEMP rtmp, TEMP ktmp);
24374 format %{ "mask_not_LT8 $dst, $src, $cnt \t!using $ktmp and $rtmp as TEMP" %}
24375 ins_encode %{
24376 uint masklen = Matcher::vector_length(this);
24377 __ knot(masklen, $dst$$KRegister, $src$$KRegister, $ktmp$$KRegister, $rtmp$$Register);
24378 %}
24379 ins_pipe( pipe_slow );
24380 %}
24381
24382 instruct mask_not_imm(kReg dst, kReg src, immI_M1 cnt) %{
24383 predicate((Matcher::vector_length(n) == 8 && VM_Version::supports_avx512dq()) ||
24384 (Matcher::vector_length(n) == 16) ||
24385 (Matcher::vector_length(n) > 16 && VM_Version::supports_avx512bw()));
24386 match(Set dst (XorVMask src (MaskAll cnt)));
24387 format %{ "mask_not $dst, $src, $cnt \t! mask not operation" %}
24388 ins_encode %{
24389 uint masklen = Matcher::vector_length(this);
24390 __ knot(masklen, $dst$$KRegister, $src$$KRegister);
24391 %}
24392 ins_pipe( pipe_slow );
24393 %}
24394
24395 instruct long_to_maskLE8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2) %{
24396 predicate(n->bottom_type()->isa_pvectmask() == nullptr && Matcher::vector_length(n) <= 8);
24397 match(Set dst (VectorLongToMask src));
24398 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2);
24399 format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2" %}
24400 ins_encode %{
24401 int mask_len = Matcher::vector_length(this);
24402 int vec_enc = vector_length_encoding(mask_len);
24403 __ vector_long_to_maskvec($dst$$XMMRegister, $src$$Register, $rtmp1$$Register,
24404 $rtmp2$$Register, xnoreg, mask_len, vec_enc);
24405 %}
24406 ins_pipe( pipe_slow );
24407 %}
24408
24409
24410 instruct long_to_maskGT8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec xtmp1, rFlagsReg cr) %{
24411 predicate(n->bottom_type()->isa_pvectmask() == nullptr && Matcher::vector_length(n) > 8);
24412 match(Set dst (VectorLongToMask src));
24413 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, TEMP xtmp1, KILL cr);
24414 format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2, $xtmp1, as TEMP" %}
24415 ins_encode %{
24416 int mask_len = Matcher::vector_length(this);
24417 assert(mask_len <= 32, "invalid mask length");
24418 int vec_enc = vector_length_encoding(mask_len);
24419 __ vector_long_to_maskvec($dst$$XMMRegister, $src$$Register, $rtmp1$$Register,
24420 $rtmp2$$Register, $xtmp1$$XMMRegister, mask_len, vec_enc);
24421 %}
24422 ins_pipe( pipe_slow );
24423 %}
24424
24425 instruct long_to_mask_evex(kReg dst, rRegL src) %{
24426 predicate(n->bottom_type()->isa_pvectmask());
24427 match(Set dst (VectorLongToMask src));
24428 format %{ "long_to_mask_evex $dst, $src\t!" %}
24429 ins_encode %{
24430 __ kmov($dst$$KRegister, $src$$Register);
24431 %}
24432 ins_pipe( pipe_slow );
24433 %}
24434
24435 instruct mask_opers_evex(kReg dst, kReg src1, kReg src2, kReg kscratch) %{
24436 match(Set dst (AndVMask src1 src2));
24437 match(Set dst (OrVMask src1 src2));
24438 match(Set dst (XorVMask src1 src2));
24439 effect(TEMP kscratch);
24440 format %{ "mask_opers_evex $dst, $src1, $src2\t! using $kscratch as TEMP" %}
24441 ins_encode %{
24442 const MachNode* mask1 = static_cast<const MachNode*>(this->in(this->operand_index($src1)));
24443 const MachNode* mask2 = static_cast<const MachNode*>(this->in(this->operand_index($src2)));
24444 assert(Type::equals(mask1->bottom_type(), mask2->bottom_type()), "Mask types must be equal");
24445 uint masklen = Matcher::vector_length(this);
24446 masklen = (masklen < 16 && !VM_Version::supports_avx512dq()) ? 16 : masklen;
24447 __ masked_op(this->ideal_Opcode(), masklen, $dst$$KRegister, $src1$$KRegister, $src2$$KRegister);
24448 %}
24449 ins_pipe( pipe_slow );
24450 %}
24451
24452 instruct vternlog_reg_masked(vec dst, vec src2, vec src3, immU8 func, kReg mask) %{
24453 match(Set dst (MacroLogicV dst (Binary src2 (Binary src3 (Binary func mask)))));
24454 format %{ "vternlog_masked $dst,$src2,$src3,$func,$mask\t! vternlog masked operation" %}
24455 ins_encode %{
24456 int vlen_enc = vector_length_encoding(this);
24457 BasicType bt = Matcher::vector_element_basic_type(this);
24458 __ evpternlog($dst$$XMMRegister, $func$$constant, $mask$$KRegister,
24459 $src2$$XMMRegister, $src3$$XMMRegister, true, bt, vlen_enc);
24460 %}
24461 ins_pipe( pipe_slow );
24462 %}
24463
24464 instruct vternlogd_mem_masked(vec dst, vec src2, memory src3, immU8 func, kReg mask) %{
24465 match(Set dst (MacroLogicV dst (Binary src2 (Binary src3 (Binary func mask)))));
24466 format %{ "vternlog_masked $dst,$src2,$src3,$func,$mask\t! vternlog masked operation" %}
24467 ins_encode %{
24468 int vlen_enc = vector_length_encoding(this);
24469 BasicType bt = Matcher::vector_element_basic_type(this);
24470 __ evpternlog($dst$$XMMRegister, $func$$constant, $mask$$KRegister,
24471 $src2$$XMMRegister, $src3$$Address, true, bt, vlen_enc);
24472 %}
24473 ins_pipe( pipe_slow );
24474 %}
24475
24476 instruct castMM(kReg dst)
24477 %{
24478 match(Set dst (CastVV dst));
24479
24480 size(0);
24481 format %{ "# castVV of $dst" %}
24482 ins_encode(/* empty encoding */);
24483 ins_cost(0);
24484 ins_pipe(empty);
24485 %}
24486
24487 instruct castVV(vec dst)
24488 %{
24489 match(Set dst (CastVV dst));
24490
24491 size(0);
24492 format %{ "# castVV of $dst" %}
24493 ins_encode(/* empty encoding */);
24494 ins_cost(0);
24495 ins_pipe(empty);
24496 %}
24497
24498 instruct castVVLeg(legVec dst)
24499 %{
24500 match(Set dst (CastVV dst));
24501
24502 size(0);
24503 format %{ "# castVV of $dst" %}
24504 ins_encode(/* empty encoding */);
24505 ins_cost(0);
24506 ins_pipe(empty);
24507 %}
24508
24509 instruct FloatClassCheck_reg_reg_vfpclass(rRegI dst, regF src, kReg ktmp, rFlagsReg cr)
24510 %{
24511 match(Set dst (IsInfiniteF src));
24512 effect(TEMP ktmp, KILL cr);
24513 format %{ "float_class_check $dst, $src" %}
24514 ins_encode %{
24515 __ vfpclassss($ktmp$$KRegister, $src$$XMMRegister, 0x18);
24516 __ kmovbl($dst$$Register, $ktmp$$KRegister);
24517 %}
24518 ins_pipe(pipe_slow);
24519 %}
24520
24521 instruct DoubleClassCheck_reg_reg_vfpclass(rRegI dst, regD src, kReg ktmp, rFlagsReg cr)
24522 %{
24523 match(Set dst (IsInfiniteD src));
24524 effect(TEMP ktmp, KILL cr);
24525 format %{ "double_class_check $dst, $src" %}
24526 ins_encode %{
24527 __ vfpclasssd($ktmp$$KRegister, $src$$XMMRegister, 0x18);
24528 __ kmovbl($dst$$Register, $ktmp$$KRegister);
24529 %}
24530 ins_pipe(pipe_slow);
24531 %}
24532
24533 instruct vector_addsub_saturating_subword_reg(vec dst, vec src1, vec src2)
24534 %{
24535 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24536 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
24537 match(Set dst (SaturatingAddV src1 src2));
24538 match(Set dst (SaturatingSubV src1 src2));
24539 format %{ "vector_addsub_saturating_subword $dst, $src1, $src2" %}
24540 ins_encode %{
24541 int vlen_enc = vector_length_encoding(this);
24542 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24543 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24544 $src1$$XMMRegister, $src2$$XMMRegister, false, vlen_enc);
24545 %}
24546 ins_pipe(pipe_slow);
24547 %}
24548
24549 instruct vector_addsub_saturating_unsigned_subword_reg(vec dst, vec src1, vec src2)
24550 %{
24551 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24552 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
24553 match(Set dst (SaturatingAddV src1 src2));
24554 match(Set dst (SaturatingSubV src1 src2));
24555 format %{ "vector_addsub_saturating_unsigned_subword $dst, $src1, $src2" %}
24556 ins_encode %{
24557 int vlen_enc = vector_length_encoding(this);
24558 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24559 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24560 $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24561 %}
24562 ins_pipe(pipe_slow);
24563 %}
24564
24565 instruct vector_addsub_saturating_reg_evex(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2)
24566 %{
24567 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24568 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned() &&
24569 (Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()));
24570 match(Set dst (SaturatingAddV src1 src2));
24571 match(Set dst (SaturatingSubV src1 src2));
24572 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2);
24573 format %{ "vector_addsub_saturating_evex $dst, $src1, $src2 \t! using $xtmp1, $xtmp2, $ktmp1 and $ktmp2 as TEMP" %}
24574 ins_encode %{
24575 int vlen_enc = vector_length_encoding(this);
24576 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24577 __ vector_addsub_dq_saturating_evex(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24578 $src1$$XMMRegister, $src2$$XMMRegister,
24579 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister,
24580 $ktmp1$$KRegister, $ktmp2$$KRegister, vlen_enc);
24581 %}
24582 ins_pipe(pipe_slow);
24583 %}
24584
24585 instruct vector_addsub_saturating_reg_avx(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4)
24586 %{
24587 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24588 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned() &&
24589 Matcher::vector_length_in_bytes(n) <= 32 && !VM_Version::supports_avx512vl());
24590 match(Set dst (SaturatingAddV src1 src2));
24591 match(Set dst (SaturatingSubV src1 src2));
24592 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4);
24593 format %{ "vector_addsub_saturating_avx $dst, $src1, $src2 \t! using $xtmp1, $xtmp2, $xtmp3 and $xtmp4 as TEMP" %}
24594 ins_encode %{
24595 int vlen_enc = vector_length_encoding(this);
24596 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24597 __ vector_addsub_dq_saturating_avx(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister, $src1$$XMMRegister,
24598 $src2$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister,
24599 $xtmp3$$XMMRegister, $xtmp4$$XMMRegister, vlen_enc);
24600 %}
24601 ins_pipe(pipe_slow);
24602 %}
24603
24604 instruct vector_add_saturating_unsigned_reg_evex(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, kReg ktmp)
24605 %{
24606 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24607 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24608 (Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()));
24609 match(Set dst (SaturatingAddV src1 src2));
24610 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp);
24611 format %{ "vector_add_saturating_unsigned_evex $dst, $src1, $src2 \t! using $xtmp1, $xtmp2 and $ktmp as TEMP" %}
24612 ins_encode %{
24613 int vlen_enc = vector_length_encoding(this);
24614 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24615 __ vector_add_dq_saturating_unsigned_evex(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
24616 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp$$KRegister, vlen_enc);
24617 %}
24618 ins_pipe(pipe_slow);
24619 %}
24620
24621 instruct vector_add_saturating_unsigned_reg_avx(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, vec xtmp3)
24622 %{
24623 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24624 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24625 Matcher::vector_length_in_bytes(n) <= 32 && !VM_Version::supports_avx512vl());
24626 match(Set dst (SaturatingAddV src1 src2));
24627 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3);
24628 format %{ "vector_add_saturating_unsigned_avx $dst, $src1, $src2 \t! using $xtmp1, $xtmp2 and $xtmp3 as TEMP" %}
24629 ins_encode %{
24630 int vlen_enc = vector_length_encoding(this);
24631 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24632 __ vector_add_dq_saturating_unsigned_avx(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
24633 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, vlen_enc);
24634 %}
24635 ins_pipe(pipe_slow);
24636 %}
24637
24638 instruct vector_sub_saturating_unsigned_reg_evex(vec dst, vec src1, vec src2, kReg ktmp)
24639 %{
24640 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24641 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24642 (Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()));
24643 match(Set dst (SaturatingSubV src1 src2));
24644 effect(TEMP ktmp);
24645 format %{ "vector_sub_saturating_unsigned_evex $dst, $src1, $src2 \t! using $ktmp as TEMP" %}
24646 ins_encode %{
24647 int vlen_enc = vector_length_encoding(this);
24648 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24649 __ vector_sub_dq_saturating_unsigned_evex(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister,
24650 $src2$$XMMRegister, $ktmp$$KRegister, vlen_enc);
24651 %}
24652 ins_pipe(pipe_slow);
24653 %}
24654
24655 instruct vector_sub_saturating_unsigned_reg_avx(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2)
24656 %{
24657 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24658 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24659 Matcher::vector_length_in_bytes(n) <= 32 && !VM_Version::supports_avx512vl());
24660 match(Set dst (SaturatingSubV src1 src2));
24661 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
24662 format %{ "vector_sub_saturating_unsigned_avx $dst, $src1, $src2 \t! using $xtmp1 and $xtmp2 as TEMP" %}
24663 ins_encode %{
24664 int vlen_enc = vector_length_encoding(this);
24665 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24666 __ vector_sub_dq_saturating_unsigned_avx(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
24667 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
24668 %}
24669 ins_pipe(pipe_slow);
24670 %}
24671
24672 instruct vector_addsub_saturating_subword_mem(vec dst, vec src1, memory src2)
24673 %{
24674 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24675 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
24676 match(Set dst (SaturatingAddV src1 (LoadVector src2)));
24677 match(Set dst (SaturatingSubV src1 (LoadVector src2)));
24678 format %{ "vector_addsub_saturating_subword $dst, $src1, $src2" %}
24679 ins_encode %{
24680 int vlen_enc = vector_length_encoding(this);
24681 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24682 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24683 $src1$$XMMRegister, $src2$$Address, false, vlen_enc);
24684 %}
24685 ins_pipe(pipe_slow);
24686 %}
24687
24688 instruct vector_addsub_saturating_unsigned_subword_mem(vec dst, vec src1, memory src2)
24689 %{
24690 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24691 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
24692 match(Set dst (SaturatingAddV src1 (LoadVector src2)));
24693 match(Set dst (SaturatingSubV src1 (LoadVector src2)));
24694 format %{ "vector_addsub_saturating_unsigned_subword $dst, $src1, $src2" %}
24695 ins_encode %{
24696 int vlen_enc = vector_length_encoding(this);
24697 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24698 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24699 $src1$$XMMRegister, $src2$$Address, true, vlen_enc);
24700 %}
24701 ins_pipe(pipe_slow);
24702 %}
24703
24704 instruct vector_addsub_saturating_subword_masked_reg(vec dst, vec src, kReg mask) %{
24705 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24706 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
24707 match(Set dst (SaturatingAddV (Binary dst src) mask));
24708 match(Set dst (SaturatingSubV (Binary dst src) mask));
24709 format %{ "vector_addsub_saturating_subword_masked $dst, $mask, $src" %}
24710 ins_encode %{
24711 int vlen_enc = vector_length_encoding(this);
24712 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24713 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
24714 $dst$$XMMRegister, $src$$XMMRegister, false, true, vlen_enc);
24715 %}
24716 ins_pipe( pipe_slow );
24717 %}
24718
24719 instruct vector_addsub_saturating_unsigned_subword_masked_reg(vec dst, vec src, kReg mask) %{
24720 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24721 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
24722 match(Set dst (SaturatingAddV (Binary dst src) mask));
24723 match(Set dst (SaturatingSubV (Binary dst src) mask));
24724 format %{ "vector_addsub_saturating_unsigned_subword_masked $dst, $mask, $src" %}
24725 ins_encode %{
24726 int vlen_enc = vector_length_encoding(this);
24727 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24728 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
24729 $dst$$XMMRegister, $src$$XMMRegister, true, true, vlen_enc);
24730 %}
24731 ins_pipe( pipe_slow );
24732 %}
24733
24734 instruct vector_addsub_saturating_subword_masked_mem(vec dst, memory src, kReg mask) %{
24735 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24736 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
24737 match(Set dst (SaturatingAddV (Binary dst (LoadVector src)) mask));
24738 match(Set dst (SaturatingSubV (Binary dst (LoadVector src)) mask));
24739 format %{ "vector_addsub_saturating_subword_masked $dst, $mask, $src" %}
24740 ins_encode %{
24741 int vlen_enc = vector_length_encoding(this);
24742 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24743 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
24744 $dst$$XMMRegister, $src$$Address, false, true, vlen_enc);
24745 %}
24746 ins_pipe( pipe_slow );
24747 %}
24748
24749 instruct vector_addsub_saturating_unsigned_subword_masked_mem(vec dst, memory src, kReg mask) %{
24750 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24751 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
24752 match(Set dst (SaturatingAddV (Binary dst (LoadVector src)) mask));
24753 match(Set dst (SaturatingSubV (Binary dst (LoadVector src)) mask));
24754 format %{ "vector_addsub_saturating_unsigned_subword_masked $dst, $mask, $src" %}
24755 ins_encode %{
24756 int vlen_enc = vector_length_encoding(this);
24757 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24758 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
24759 $dst$$XMMRegister, $src$$Address, true, true, vlen_enc);
24760 %}
24761 ins_pipe( pipe_slow );
24762 %}
24763
24764 instruct vector_selectfrom_twovectors_reg_evex(vec index, vec src1, vec src2)
24765 %{
24766 match(Set index (SelectFromTwoVector (Binary index src1) src2));
24767 format %{ "select_from_two_vector $index, $src1, $src2 \t!" %}
24768 ins_encode %{
24769 int vlen_enc = vector_length_encoding(this);
24770 BasicType bt = Matcher::vector_element_basic_type(this);
24771 __ select_from_two_vectors_evex(bt, $index$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
24772 %}
24773 ins_pipe(pipe_slow);
24774 %}
24775
24776 instruct reinterpretS2HF(regF dst, rRegI src)
24777 %{
24778 match(Set dst (ReinterpretS2HF src));
24779 format %{ "evmovw $dst, $src" %}
24780 ins_encode %{
24781 __ evmovw($dst$$XMMRegister, $src$$Register);
24782 %}
24783 ins_pipe(pipe_slow);
24784 %}
24785
24786 instruct reinterpretHF2S(rRegI dst, regF src)
24787 %{
24788 match(Set dst (ReinterpretHF2S src));
24789 format %{ "evmovw $dst, $src" %}
24790 ins_encode %{
24791 __ evmovw($dst$$Register, $src$$XMMRegister);
24792 __ narrow_subword_type($dst$$Register, T_SHORT);
24793 %}
24794 ins_pipe(pipe_slow);
24795 %}
24796
24797 instruct convF2HFAndS2HF(regF dst, regF src)
24798 %{
24799 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
24800 format %{ "convF2HFAndS2HF $dst, $src" %}
24801 ins_encode %{
24802 __ vcvtps2ph($dst$$XMMRegister, $src$$XMMRegister, 0x04, Assembler::AVX_128bit);
24803 %}
24804 ins_pipe(pipe_slow);
24805 %}
24806
24807 instruct convHF2SAndHF2F(regF dst, regF src)
24808 %{
24809 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
24810 format %{ "convHF2SAndHF2F $dst, $src" %}
24811 ins_encode %{
24812 __ vcvtph2ps($dst$$XMMRegister, $src$$XMMRegister, Assembler::AVX_128bit);
24813 %}
24814 ins_pipe(pipe_slow);
24815 %}
24816
24817 instruct scalar_sqrt_HF_reg(regF dst, regF src)
24818 %{
24819 match(Set dst (SqrtHF src));
24820 format %{ "scalar_sqrt_fp16 $dst, $src" %}
24821 ins_encode %{
24822 __ vsqrtsh($dst$$XMMRegister, $src$$XMMRegister);
24823 %}
24824 ins_pipe(pipe_slow);
24825 %}
24826
24827 instruct scalar_binOps_HF_reg(regF dst, regF src1, regF src2)
24828 %{
24829 match(Set dst (AddHF src1 src2));
24830 match(Set dst (DivHF src1 src2));
24831 match(Set dst (MulHF src1 src2));
24832 match(Set dst (SubHF src1 src2));
24833 format %{ "scalar_binop_fp16 $dst, $src1, $src2" %}
24834 ins_encode %{
24835 int opcode = this->ideal_Opcode();
24836 __ efp16sh(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
24837 %}
24838 ins_pipe(pipe_slow);
24839 %}
24840
24841 instruct scalar_minmax_HF_reg_avx10_2(regF dst, regF src1, regF src2)
24842 %{
24843 predicate(VM_Version::supports_avx10_2());
24844 match(Set dst (MaxHF src1 src2));
24845 match(Set dst (MinHF src1 src2));
24846
24847 format %{ "scalar_min_max_fp16 $dst, $src1, $src2" %}
24848 ins_encode %{
24849 int opcode = this->ideal_Opcode();
24850 __ sminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, k0);
24851 %}
24852 ins_pipe( pipe_slow );
24853 %}
24854
24855 instruct scalar_minmax_HF_reg(regF dst, regF src1, regF src2, kReg ktmp, regF xtmp1, regF xtmp2)
24856 %{
24857 predicate(!VM_Version::supports_avx10_2());
24858 match(Set dst (MaxHF src1 src2));
24859 match(Set dst (MinHF src1 src2));
24860 effect(TEMP_DEF dst, TEMP ktmp, TEMP xtmp1, TEMP xtmp2);
24861
24862 format %{ "scalar_min_max_fp16 $dst, $src1, $src2\t using $ktmp, $xtmp1 and $xtmp2 as TEMP" %}
24863 ins_encode %{
24864 int opcode = this->ideal_Opcode();
24865 __ sminmax_fp16(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $ktmp$$KRegister,
24866 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
24867 %}
24868 ins_pipe( pipe_slow );
24869 %}
24870
24871 instruct scalar_fma_HF_reg(regF dst, regF src1, regF src2)
24872 %{
24873 match(Set dst (FmaHF src2 (Binary dst src1)));
24874 effect(DEF dst);
24875 format %{ "scalar_fma_fp16 $dst, $src1, $src2\t# $dst = $dst * $src1 + $src2 fma packedH" %}
24876 ins_encode %{
24877 __ vfmadd132sh($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister);
24878 %}
24879 ins_pipe( pipe_slow );
24880 %}
24881
24882
24883 instruct vector_sqrt_HF_reg(vec dst, vec src)
24884 %{
24885 match(Set dst (SqrtVHF src));
24886 format %{ "vector_sqrt_fp16 $dst, $src" %}
24887 ins_encode %{
24888 int vlen_enc = vector_length_encoding(this);
24889 __ evsqrtph($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
24890 %}
24891 ins_pipe(pipe_slow);
24892 %}
24893
24894 instruct vector_sqrt_HF_mem(vec dst, memory src)
24895 %{
24896 match(Set dst (SqrtVHF (VectorReinterpret (LoadVector src))));
24897 format %{ "vector_sqrt_fp16_mem $dst, $src" %}
24898 ins_encode %{
24899 int vlen_enc = vector_length_encoding(this);
24900 __ evsqrtph($dst$$XMMRegister, $src$$Address, vlen_enc);
24901 %}
24902 ins_pipe(pipe_slow);
24903 %}
24904
24905 instruct vector_binOps_HF_reg(vec dst, vec src1, vec src2)
24906 %{
24907 match(Set dst (AddVHF src1 src2));
24908 match(Set dst (DivVHF src1 src2));
24909 match(Set dst (MulVHF src1 src2));
24910 match(Set dst (SubVHF src1 src2));
24911 format %{ "vector_binop_fp16 $dst, $src1, $src2" %}
24912 ins_encode %{
24913 int vlen_enc = vector_length_encoding(this);
24914 int opcode = this->ideal_Opcode();
24915 __ evfp16ph(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
24916 %}
24917 ins_pipe(pipe_slow);
24918 %}
24919
24920
24921 instruct vector_binOps_HF_mem(vec dst, vec src1, memory src2)
24922 %{
24923 match(Set dst (AddVHF src1 (VectorReinterpret (LoadVector src2))));
24924 match(Set dst (DivVHF src1 (VectorReinterpret (LoadVector src2))));
24925 match(Set dst (MulVHF src1 (VectorReinterpret (LoadVector src2))));
24926 match(Set dst (SubVHF src1 (VectorReinterpret (LoadVector src2))));
24927 format %{ "vector_binop_fp16_mem $dst, $src1, $src2" %}
24928 ins_encode %{
24929 int vlen_enc = vector_length_encoding(this);
24930 int opcode = this->ideal_Opcode();
24931 __ evfp16ph(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address, vlen_enc);
24932 %}
24933 ins_pipe(pipe_slow);
24934 %}
24935
24936 instruct vector_fma_HF_reg(vec dst, vec src1, vec src2)
24937 %{
24938 match(Set dst (FmaVHF src2 (Binary dst src1)));
24939 format %{ "vector_fma_fp16 $dst, $src1, $src2\t# $dst = $dst * $src1 + $src2 fma packedH" %}
24940 ins_encode %{
24941 int vlen_enc = vector_length_encoding(this);
24942 __ evfmadd132ph($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vlen_enc);
24943 %}
24944 ins_pipe( pipe_slow );
24945 %}
24946
24947 instruct vector_fma_HF_mem(vec dst, memory src1, vec src2)
24948 %{
24949 match(Set dst (FmaVHF src2 (Binary dst (VectorReinterpret (LoadVector src1)))));
24950 format %{ "vector_fma_fp16_mem $dst, $src1, $src2\t# $dst = $dst * $src1 + $src2 fma packedH" %}
24951 ins_encode %{
24952 int vlen_enc = vector_length_encoding(this);
24953 __ evfmadd132ph($dst$$XMMRegister, $src2$$XMMRegister, $src1$$Address, vlen_enc);
24954 %}
24955 ins_pipe( pipe_slow );
24956 %}
24957
24958 instruct vector_minmax_HF_mem_avx10_2(vec dst, vec src1, memory src2)
24959 %{
24960 predicate(VM_Version::supports_avx10_2());
24961 match(Set dst (MinVHF src1 (VectorReinterpret (LoadVector src2))));
24962 match(Set dst (MaxVHF src1 (VectorReinterpret (LoadVector src2))));
24963 format %{ "vector_min_max_fp16_mem $dst, $src1, $src2" %}
24964 ins_encode %{
24965 int vlen_enc = vector_length_encoding(this);
24966 int opcode = this->ideal_Opcode();
24967 __ vminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address,
24968 k0, vlen_enc);
24969 %}
24970 ins_pipe( pipe_slow );
24971 %}
24972
24973 instruct vector_minmax_HF_reg_avx10_2(vec dst, vec src1, vec src2)
24974 %{
24975 predicate(VM_Version::supports_avx10_2());
24976 match(Set dst (MinVHF src1 src2));
24977 match(Set dst (MaxVHF src1 src2));
24978 format %{ "vector_min_max_fp16 $dst, $src1, $src2" %}
24979 ins_encode %{
24980 int vlen_enc = vector_length_encoding(this);
24981 int opcode = this->ideal_Opcode();
24982 __ vminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
24983 k0, vlen_enc);
24984 %}
24985 ins_pipe( pipe_slow );
24986 %}
24987
24988 instruct vector_minmax_HF_reg(vec dst, vec src1, vec src2, kReg ktmp, vec xtmp1, vec xtmp2)
24989 %{
24990 predicate(!VM_Version::supports_avx10_2());
24991 match(Set dst (MinVHF src1 src2));
24992 match(Set dst (MaxVHF src1 src2));
24993 effect(TEMP_DEF dst, TEMP ktmp, TEMP xtmp1, TEMP xtmp2);
24994 format %{ "vector_min_max_fp16 $dst, $src1, $src2\t using $ktmp, $xtmp1 and $xtmp2 as TEMP" %}
24995 ins_encode %{
24996 int vlen_enc = vector_length_encoding(this);
24997 int opcode = this->ideal_Opcode();
24998 __ vminmax_fp16(opcode, $dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, $ktmp$$KRegister,
24999 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
25000 %}
25001 ins_pipe( pipe_slow );
25002 %}
25003
25004 //----------PEEPHOLE RULES-----------------------------------------------------
25005 // These must follow all instruction definitions as they use the names
25006 // defined in the instructions definitions.
25007 //
25008 // peeppredicate ( rule_predicate );
25009 // // the predicate unless which the peephole rule will be ignored
25010 //
25011 // peepmatch ( root_instr_name [preceding_instruction]* );
25012 //
25013 // peepprocedure ( procedure_name );
25014 // // provide a procedure name to perform the optimization, the procedure should
25015 // // reside in the architecture dependent peephole file, the method has the
25016 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
25017 // // with the arguments being the basic block, the current node index inside the
25018 // // block, the register allocator, the functions upon invoked return a new node
25019 // // defined in peepreplace, and the rules of the nodes appearing in the
25020 // // corresponding peepmatch, the function return true if successful, else
25021 // // return false
25022 //
25023 // peepconstraint %{
25024 // (instruction_number.operand_name relational_op instruction_number.operand_name
25025 // [, ...] );
25026 // // instruction numbers are zero-based using left to right order in peepmatch
25027 //
25028 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
25029 // // provide an instruction_number.operand_name for each operand that appears
25030 // // in the replacement instruction's match rule
25031 //
25032 // ---------VM FLAGS---------------------------------------------------------
25033 //
25034 // All peephole optimizations can be turned off using -XX:-OptoPeephole
25035 //
25036 // Each peephole rule is given an identifying number starting with zero and
25037 // increasing by one in the order seen by the parser. An individual peephole
25038 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
25039 // on the command-line.
25040 //
25041 // ---------CURRENT LIMITATIONS----------------------------------------------
25042 //
25043 // Only transformations inside a basic block (do we need more for peephole)
25044 //
25045 // ---------EXAMPLE----------------------------------------------------------
25046 //
25047 // // pertinent parts of existing instructions in architecture description
25048 // instruct movI(rRegI dst, rRegI src)
25049 // %{
25050 // match(Set dst (CopyI src));
25051 // %}
25052 //
25053 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
25054 // %{
25055 // match(Set dst (AddI dst src));
25056 // effect(KILL cr);
25057 // %}
25058 //
25059 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
25060 // %{
25061 // match(Set dst (AddI dst src));
25062 // %}
25063 //
25064 // 1. Simple replacement
25065 // - Only match adjacent instructions in same basic block
25066 // - Only equality constraints
25067 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
25068 // - Only one replacement instruction
25069 //
25070 // // Change (inc mov) to lea
25071 // peephole %{
25072 // // lea should only be emitted when beneficial
25073 // peeppredicate( VM_Version::supports_fast_2op_lea() );
25074 // // increment preceded by register-register move
25075 // peepmatch ( incI_rReg movI );
25076 // // require that the destination register of the increment
25077 // // match the destination register of the move
25078 // peepconstraint ( 0.dst == 1.dst );
25079 // // construct a replacement instruction that sets
25080 // // the destination to ( move's source register + one )
25081 // peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
25082 // %}
25083 //
25084 // 2. Procedural replacement
25085 // - More flexible finding relevent nodes
25086 // - More flexible constraints
25087 // - More flexible transformations
25088 // - May utilise architecture-dependent API more effectively
25089 // - Currently only one replacement instruction due to adlc parsing capabilities
25090 //
25091 // // Change (inc mov) to lea
25092 // peephole %{
25093 // // lea should only be emitted when beneficial
25094 // peeppredicate( VM_Version::supports_fast_2op_lea() );
25095 // // the rule numbers of these nodes inside are passed into the function below
25096 // peepmatch ( incI_rReg movI );
25097 // // the method that takes the responsibility of transformation
25098 // peepprocedure ( inc_mov_to_lea );
25099 // // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
25100 // // node is passed into the function above
25101 // peepreplace ( leaI_rReg_immI() );
25102 // %}
25103
25104 // These instructions is not matched by the matcher but used by the peephole
25105 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
25106 %{
25107 predicate(false);
25108 match(Set dst (AddI src1 src2));
25109 format %{ "leal $dst, [$src1 + $src2]" %}
25110 ins_encode %{
25111 Register dst = $dst$$Register;
25112 Register src1 = $src1$$Register;
25113 Register src2 = $src2$$Register;
25114 if (src1 != rbp && src1 != r13) {
25115 __ leal(dst, Address(src1, src2, Address::times_1));
25116 } else {
25117 assert(src2 != rbp && src2 != r13, "");
25118 __ leal(dst, Address(src2, src1, Address::times_1));
25119 }
25120 %}
25121 ins_pipe(ialu_reg_reg);
25122 %}
25123
25124 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
25125 %{
25126 predicate(false);
25127 match(Set dst (AddI src1 src2));
25128 format %{ "leal $dst, [$src1 + $src2]" %}
25129 ins_encode %{
25130 __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
25131 %}
25132 ins_pipe(ialu_reg_reg);
25133 %}
25134
25135 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
25136 %{
25137 predicate(false);
25138 match(Set dst (LShiftI src shift));
25139 format %{ "leal $dst, [$src << $shift]" %}
25140 ins_encode %{
25141 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
25142 Register src = $src$$Register;
25143 if (scale == Address::times_2 && src != rbp && src != r13) {
25144 __ leal($dst$$Register, Address(src, src, Address::times_1));
25145 } else {
25146 __ leal($dst$$Register, Address(noreg, src, scale));
25147 }
25148 %}
25149 ins_pipe(ialu_reg_reg);
25150 %}
25151
25152 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
25153 %{
25154 predicate(false);
25155 match(Set dst (AddL src1 src2));
25156 format %{ "leaq $dst, [$src1 + $src2]" %}
25157 ins_encode %{
25158 Register dst = $dst$$Register;
25159 Register src1 = $src1$$Register;
25160 Register src2 = $src2$$Register;
25161 if (src1 != rbp && src1 != r13) {
25162 __ leaq(dst, Address(src1, src2, Address::times_1));
25163 } else {
25164 assert(src2 != rbp && src2 != r13, "");
25165 __ leaq(dst, Address(src2, src1, Address::times_1));
25166 }
25167 %}
25168 ins_pipe(ialu_reg_reg);
25169 %}
25170
25171 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
25172 %{
25173 predicate(false);
25174 match(Set dst (AddL src1 src2));
25175 format %{ "leaq $dst, [$src1 + $src2]" %}
25176 ins_encode %{
25177 __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
25178 %}
25179 ins_pipe(ialu_reg_reg);
25180 %}
25181
25182 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
25183 %{
25184 predicate(false);
25185 match(Set dst (LShiftL src shift));
25186 format %{ "leaq $dst, [$src << $shift]" %}
25187 ins_encode %{
25188 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
25189 Register src = $src$$Register;
25190 if (scale == Address::times_2 && src != rbp && src != r13) {
25191 __ leaq($dst$$Register, Address(src, src, Address::times_1));
25192 } else {
25193 __ leaq($dst$$Register, Address(noreg, src, scale));
25194 }
25195 %}
25196 ins_pipe(ialu_reg_reg);
25197 %}
25198
25199 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
25200 // sal}) with lea instructions. The {add, sal} rules are beneficial in
25201 // processors with at least partial ALU support for lea
25202 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
25203 // beneficial for processors with full ALU support
25204 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
25205
25206 peephole
25207 %{
25208 peeppredicate(VM_Version::supports_fast_2op_lea());
25209 peepmatch (addI_rReg);
25210 peepprocedure (lea_coalesce_reg);
25211 peepreplace (leaI_rReg_rReg_peep());
25212 %}
25213
25214 peephole
25215 %{
25216 peeppredicate(VM_Version::supports_fast_2op_lea());
25217 peepmatch (addI_rReg_imm);
25218 peepprocedure (lea_coalesce_imm);
25219 peepreplace (leaI_rReg_immI_peep());
25220 %}
25221
25222 peephole
25223 %{
25224 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25225 VM_Version::is_intel_cascade_lake());
25226 peepmatch (incI_rReg);
25227 peepprocedure (lea_coalesce_imm);
25228 peepreplace (leaI_rReg_immI_peep());
25229 %}
25230
25231 peephole
25232 %{
25233 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25234 VM_Version::is_intel_cascade_lake());
25235 peepmatch (decI_rReg);
25236 peepprocedure (lea_coalesce_imm);
25237 peepreplace (leaI_rReg_immI_peep());
25238 %}
25239
25240 peephole
25241 %{
25242 peeppredicate(VM_Version::supports_fast_2op_lea());
25243 peepmatch (salI_rReg_immI2);
25244 peepprocedure (lea_coalesce_imm);
25245 peepreplace (leaI_rReg_immI2_peep());
25246 %}
25247
25248 peephole
25249 %{
25250 peeppredicate(VM_Version::supports_fast_2op_lea());
25251 peepmatch (addL_rReg);
25252 peepprocedure (lea_coalesce_reg);
25253 peepreplace (leaL_rReg_rReg_peep());
25254 %}
25255
25256 peephole
25257 %{
25258 peeppredicate(VM_Version::supports_fast_2op_lea());
25259 peepmatch (addL_rReg_imm);
25260 peepprocedure (lea_coalesce_imm);
25261 peepreplace (leaL_rReg_immL32_peep());
25262 %}
25263
25264 peephole
25265 %{
25266 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25267 VM_Version::is_intel_cascade_lake());
25268 peepmatch (incL_rReg);
25269 peepprocedure (lea_coalesce_imm);
25270 peepreplace (leaL_rReg_immL32_peep());
25271 %}
25272
25273 peephole
25274 %{
25275 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25276 VM_Version::is_intel_cascade_lake());
25277 peepmatch (decL_rReg);
25278 peepprocedure (lea_coalesce_imm);
25279 peepreplace (leaL_rReg_immL32_peep());
25280 %}
25281
25282 peephole
25283 %{
25284 peeppredicate(VM_Version::supports_fast_2op_lea());
25285 peepmatch (salL_rReg_immI2);
25286 peepprocedure (lea_coalesce_imm);
25287 peepreplace (leaL_rReg_immI2_peep());
25288 %}
25289
25290 peephole
25291 %{
25292 peepmatch (leaPCompressedOopOffset);
25293 peepprocedure (lea_remove_redundant);
25294 %}
25295
25296 peephole
25297 %{
25298 peepmatch (leaP8Narrow);
25299 peepprocedure (lea_remove_redundant);
25300 %}
25301
25302 peephole
25303 %{
25304 peepmatch (leaP32Narrow);
25305 peepprocedure (lea_remove_redundant);
25306 %}
25307
25308 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
25309 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
25310
25311 //int variant
25312 peephole
25313 %{
25314 peepmatch (testI_reg);
25315 peepprocedure (test_may_remove);
25316 %}
25317
25318 //long variant
25319 peephole
25320 %{
25321 peepmatch (testL_reg);
25322 peepprocedure (test_may_remove);
25323 %}
25324
25325
25326 //----------SMARTSPILL RULES---------------------------------------------------
25327 // These must follow all instruction definitions as they use the names
25328 // defined in the instructions definitions.