1 //
2 // Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23 //
24
25 // X86 AMD64 Architecture Description File
26
27 //----------REGISTER DEFINITION BLOCK------------------------------------------
28 // This information is used by the matcher and the register allocator to
29 // describe individual registers and classes of registers within the target
30 // architecture.
31
32 register %{
33 //----------Architecture Description Register Definitions----------------------
34 // General Registers
35 // "reg_def" name ( register save type, C convention save type,
36 // ideal register type, encoding );
37 // Register Save Types:
38 //
39 // NS = No-Save: The register allocator assumes that these registers
40 // can be used without saving upon entry to the method, &
41 // that they do not need to be saved at call sites.
42 //
43 // SOC = Save-On-Call: The register allocator assumes that these registers
44 // can be used without saving upon entry to the method,
45 // but that they must be saved at call sites.
46 //
47 // SOE = Save-On-Entry: The register allocator assumes that these registers
48 // must be saved before using them upon entry to the
49 // method, but they do not need to be saved at call
50 // sites.
51 //
52 // AS = Always-Save: The register allocator assumes that these registers
53 // must be saved before using them upon entry to the
54 // method, & that they must be saved at call sites.
55 //
56 // Ideal Register Type is used to determine how to save & restore a
57 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
58 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
59 //
60 // The encoding number is the actual bit-pattern placed into the opcodes.
61
62 // General Registers
63 // R8-R15 must be encoded with REX. (RSP, RBP, RSI, RDI need REX when
64 // used as byte registers)
65
66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
69
70 reg_def RAX (SOC, SOC, Op_RegI, 0, rax->as_VMReg());
71 reg_def RAX_H(SOC, SOC, Op_RegI, 0, rax->as_VMReg()->next());
72
73 reg_def RCX (SOC, SOC, Op_RegI, 1, rcx->as_VMReg());
74 reg_def RCX_H(SOC, SOC, Op_RegI, 1, rcx->as_VMReg()->next());
75
76 reg_def RDX (SOC, SOC, Op_RegI, 2, rdx->as_VMReg());
77 reg_def RDX_H(SOC, SOC, Op_RegI, 2, rdx->as_VMReg()->next());
78
79 reg_def RBX (SOC, SOE, Op_RegI, 3, rbx->as_VMReg());
80 reg_def RBX_H(SOC, SOE, Op_RegI, 3, rbx->as_VMReg()->next());
81
82 reg_def RSP (NS, NS, Op_RegI, 4, rsp->as_VMReg());
83 reg_def RSP_H(NS, NS, Op_RegI, 4, rsp->as_VMReg()->next());
84
85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
86 reg_def RBP (NS, SOE, Op_RegI, 5, rbp->as_VMReg());
87 reg_def RBP_H(NS, SOE, Op_RegI, 5, rbp->as_VMReg()->next());
88
89 #ifdef _WIN64
90
91 reg_def RSI (SOC, SOE, Op_RegI, 6, rsi->as_VMReg());
92 reg_def RSI_H(SOC, SOE, Op_RegI, 6, rsi->as_VMReg()->next());
93
94 reg_def RDI (SOC, SOE, Op_RegI, 7, rdi->as_VMReg());
95 reg_def RDI_H(SOC, SOE, Op_RegI, 7, rdi->as_VMReg()->next());
96
97 #else
98
99 reg_def RSI (SOC, SOC, Op_RegI, 6, rsi->as_VMReg());
100 reg_def RSI_H(SOC, SOC, Op_RegI, 6, rsi->as_VMReg()->next());
101
102 reg_def RDI (SOC, SOC, Op_RegI, 7, rdi->as_VMReg());
103 reg_def RDI_H(SOC, SOC, Op_RegI, 7, rdi->as_VMReg()->next());
104
105 #endif
106
107 reg_def R8 (SOC, SOC, Op_RegI, 8, r8->as_VMReg());
108 reg_def R8_H (SOC, SOC, Op_RegI, 8, r8->as_VMReg()->next());
109
110 reg_def R9 (SOC, SOC, Op_RegI, 9, r9->as_VMReg());
111 reg_def R9_H (SOC, SOC, Op_RegI, 9, r9->as_VMReg()->next());
112
113 reg_def R10 (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
115
116 reg_def R11 (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
118
119 reg_def R12 (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
121
122 reg_def R13 (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
124
125 reg_def R14 (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
127
128 reg_def R15 (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
130
131 reg_def R16 (SOC, SOC, Op_RegI, 16, r16->as_VMReg());
132 reg_def R16_H(SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
133
134 reg_def R17 (SOC, SOC, Op_RegI, 17, r17->as_VMReg());
135 reg_def R17_H(SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
136
137 reg_def R18 (SOC, SOC, Op_RegI, 18, r18->as_VMReg());
138 reg_def R18_H(SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
139
140 reg_def R19 (SOC, SOC, Op_RegI, 19, r19->as_VMReg());
141 reg_def R19_H(SOC, SOC, Op_RegI, 19, r19->as_VMReg()->next());
142
143 reg_def R20 (SOC, SOC, Op_RegI, 20, r20->as_VMReg());
144 reg_def R20_H(SOC, SOC, Op_RegI, 20, r20->as_VMReg()->next());
145
146 reg_def R21 (SOC, SOC, Op_RegI, 21, r21->as_VMReg());
147 reg_def R21_H(SOC, SOC, Op_RegI, 21, r21->as_VMReg()->next());
148
149 reg_def R22 (SOC, SOC, Op_RegI, 22, r22->as_VMReg());
150 reg_def R22_H(SOC, SOC, Op_RegI, 22, r22->as_VMReg()->next());
151
152 reg_def R23 (SOC, SOC, Op_RegI, 23, r23->as_VMReg());
153 reg_def R23_H(SOC, SOC, Op_RegI, 23, r23->as_VMReg()->next());
154
155 reg_def R24 (SOC, SOC, Op_RegI, 24, r24->as_VMReg());
156 reg_def R24_H(SOC, SOC, Op_RegI, 24, r24->as_VMReg()->next());
157
158 reg_def R25 (SOC, SOC, Op_RegI, 25, r25->as_VMReg());
159 reg_def R25_H(SOC, SOC, Op_RegI, 25, r25->as_VMReg()->next());
160
161 reg_def R26 (SOC, SOC, Op_RegI, 26, r26->as_VMReg());
162 reg_def R26_H(SOC, SOC, Op_RegI, 26, r26->as_VMReg()->next());
163
164 reg_def R27 (SOC, SOC, Op_RegI, 27, r27->as_VMReg());
165 reg_def R27_H(SOC, SOC, Op_RegI, 27, r27->as_VMReg()->next());
166
167 reg_def R28 (SOC, SOC, Op_RegI, 28, r28->as_VMReg());
168 reg_def R28_H(SOC, SOC, Op_RegI, 28, r28->as_VMReg()->next());
169
170 reg_def R29 (SOC, SOC, Op_RegI, 29, r29->as_VMReg());
171 reg_def R29_H(SOC, SOC, Op_RegI, 29, r29->as_VMReg()->next());
172
173 reg_def R30 (SOC, SOC, Op_RegI, 30, r30->as_VMReg());
174 reg_def R30_H(SOC, SOC, Op_RegI, 30, r30->as_VMReg()->next());
175
176 reg_def R31 (SOC, SOC, Op_RegI, 31, r31->as_VMReg());
177 reg_def R31_H(SOC, SOC, Op_RegI, 31, r31->as_VMReg()->next());
178
179 // Floating Point Registers
180
181 // Specify priority of register selection within phases of register
182 // allocation. Highest priority is first. A useful heuristic is to
183 // give registers a low priority when they are required by machine
184 // instructions, like EAX and EDX on I486, and choose no-save registers
185 // before save-on-call, & save-on-call before save-on-entry. Registers
186 // which participate in fixed calling sequences should come last.
187 // Registers which are used as pairs must fall on an even boundary.
188
189 alloc_class chunk0(R10, R10_H,
190 R11, R11_H,
191 R8, R8_H,
192 R9, R9_H,
193 R12, R12_H,
194 RCX, RCX_H,
195 RBX, RBX_H,
196 RDI, RDI_H,
197 RDX, RDX_H,
198 RSI, RSI_H,
199 RAX, RAX_H,
200 RBP, RBP_H,
201 R13, R13_H,
202 R14, R14_H,
203 R15, R15_H,
204 R16, R16_H,
205 R17, R17_H,
206 R18, R18_H,
207 R19, R19_H,
208 R20, R20_H,
209 R21, R21_H,
210 R22, R22_H,
211 R23, R23_H,
212 R24, R24_H,
213 R25, R25_H,
214 R26, R26_H,
215 R27, R27_H,
216 R28, R28_H,
217 R29, R29_H,
218 R30, R30_H,
219 R31, R31_H,
220 RSP, RSP_H);
221
222 // XMM registers. 512-bit registers or 8 words each, labeled (a)-p.
223 // Word a in each register holds a Float, words ab hold a Double.
224 // The whole registers are used in SSE4.2 version intrinsics,
225 // array copy stubs and superword operations (see UseSSE42Intrinsics,
226 // UseXMMForArrayCopy and UseSuperword flags).
227 // For pre EVEX enabled architectures:
228 // XMM8-XMM15 must be encoded with REX (VEX for UseAVX)
229 // For EVEX enabled architectures:
230 // XMM8-XMM31 must be encoded with REX (EVEX for UseAVX).
231 //
232 // Linux ABI: No register preserved across function calls
233 // XMM0-XMM7 might hold parameters
234 // Windows ABI: XMM6-XMM15 preserved across function calls
235 // XMM0-XMM3 might hold parameters
236
237 reg_def XMM0 ( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg());
238 reg_def XMM0b( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(1));
239 reg_def XMM0c( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(2));
240 reg_def XMM0d( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(3));
241 reg_def XMM0e( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(4));
242 reg_def XMM0f( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(5));
243 reg_def XMM0g( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(6));
244 reg_def XMM0h( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(7));
245 reg_def XMM0i( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(8));
246 reg_def XMM0j( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(9));
247 reg_def XMM0k( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(10));
248 reg_def XMM0l( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(11));
249 reg_def XMM0m( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(12));
250 reg_def XMM0n( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(13));
251 reg_def XMM0o( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(14));
252 reg_def XMM0p( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next(15));
253
254 reg_def XMM1 ( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg());
255 reg_def XMM1b( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(1));
256 reg_def XMM1c( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(2));
257 reg_def XMM1d( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(3));
258 reg_def XMM1e( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(4));
259 reg_def XMM1f( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(5));
260 reg_def XMM1g( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(6));
261 reg_def XMM1h( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(7));
262 reg_def XMM1i( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(8));
263 reg_def XMM1j( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(9));
264 reg_def XMM1k( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(10));
265 reg_def XMM1l( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(11));
266 reg_def XMM1m( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(12));
267 reg_def XMM1n( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(13));
268 reg_def XMM1o( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(14));
269 reg_def XMM1p( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next(15));
270
271 reg_def XMM2 ( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg());
272 reg_def XMM2b( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(1));
273 reg_def XMM2c( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(2));
274 reg_def XMM2d( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(3));
275 reg_def XMM2e( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(4));
276 reg_def XMM2f( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(5));
277 reg_def XMM2g( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(6));
278 reg_def XMM2h( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(7));
279 reg_def XMM2i( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(8));
280 reg_def XMM2j( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(9));
281 reg_def XMM2k( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(10));
282 reg_def XMM2l( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(11));
283 reg_def XMM2m( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(12));
284 reg_def XMM2n( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(13));
285 reg_def XMM2o( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(14));
286 reg_def XMM2p( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next(15));
287
288 reg_def XMM3 ( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg());
289 reg_def XMM3b( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(1));
290 reg_def XMM3c( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(2));
291 reg_def XMM3d( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(3));
292 reg_def XMM3e( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(4));
293 reg_def XMM3f( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(5));
294 reg_def XMM3g( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(6));
295 reg_def XMM3h( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(7));
296 reg_def XMM3i( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(8));
297 reg_def XMM3j( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(9));
298 reg_def XMM3k( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(10));
299 reg_def XMM3l( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(11));
300 reg_def XMM3m( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(12));
301 reg_def XMM3n( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(13));
302 reg_def XMM3o( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(14));
303 reg_def XMM3p( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next(15));
304
305 reg_def XMM4 ( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg());
306 reg_def XMM4b( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(1));
307 reg_def XMM4c( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(2));
308 reg_def XMM4d( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(3));
309 reg_def XMM4e( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(4));
310 reg_def XMM4f( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(5));
311 reg_def XMM4g( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(6));
312 reg_def XMM4h( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(7));
313 reg_def XMM4i( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(8));
314 reg_def XMM4j( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(9));
315 reg_def XMM4k( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(10));
316 reg_def XMM4l( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(11));
317 reg_def XMM4m( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(12));
318 reg_def XMM4n( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(13));
319 reg_def XMM4o( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(14));
320 reg_def XMM4p( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next(15));
321
322 reg_def XMM5 ( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg());
323 reg_def XMM5b( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(1));
324 reg_def XMM5c( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(2));
325 reg_def XMM5d( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(3));
326 reg_def XMM5e( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(4));
327 reg_def XMM5f( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(5));
328 reg_def XMM5g( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(6));
329 reg_def XMM5h( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(7));
330 reg_def XMM5i( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(8));
331 reg_def XMM5j( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(9));
332 reg_def XMM5k( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(10));
333 reg_def XMM5l( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(11));
334 reg_def XMM5m( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(12));
335 reg_def XMM5n( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(13));
336 reg_def XMM5o( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(14));
337 reg_def XMM5p( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next(15));
338
339 reg_def XMM6 ( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg());
340 reg_def XMM6b( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(1));
341 reg_def XMM6c( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(2));
342 reg_def XMM6d( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(3));
343 reg_def XMM6e( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(4));
344 reg_def XMM6f( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(5));
345 reg_def XMM6g( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(6));
346 reg_def XMM6h( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(7));
347 reg_def XMM6i( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(8));
348 reg_def XMM6j( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(9));
349 reg_def XMM6k( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(10));
350 reg_def XMM6l( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(11));
351 reg_def XMM6m( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(12));
352 reg_def XMM6n( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(13));
353 reg_def XMM6o( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(14));
354 reg_def XMM6p( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next(15));
355
356 reg_def XMM7 ( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg());
357 reg_def XMM7b( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(1));
358 reg_def XMM7c( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(2));
359 reg_def XMM7d( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(3));
360 reg_def XMM7e( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(4));
361 reg_def XMM7f( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(5));
362 reg_def XMM7g( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(6));
363 reg_def XMM7h( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(7));
364 reg_def XMM7i( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(8));
365 reg_def XMM7j( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(9));
366 reg_def XMM7k( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(10));
367 reg_def XMM7l( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(11));
368 reg_def XMM7m( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(12));
369 reg_def XMM7n( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(13));
370 reg_def XMM7o( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(14));
371 reg_def XMM7p( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next(15));
372
373 reg_def XMM8 ( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg());
374 reg_def XMM8b( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(1));
375 reg_def XMM8c( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(2));
376 reg_def XMM8d( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(3));
377 reg_def XMM8e( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(4));
378 reg_def XMM8f( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(5));
379 reg_def XMM8g( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(6));
380 reg_def XMM8h( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(7));
381 reg_def XMM8i( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(8));
382 reg_def XMM8j( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(9));
383 reg_def XMM8k( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(10));
384 reg_def XMM8l( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(11));
385 reg_def XMM8m( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(12));
386 reg_def XMM8n( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(13));
387 reg_def XMM8o( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(14));
388 reg_def XMM8p( SOC, SOC, Op_RegF, 8, xmm8->as_VMReg()->next(15));
389
390 reg_def XMM9 ( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg());
391 reg_def XMM9b( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(1));
392 reg_def XMM9c( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(2));
393 reg_def XMM9d( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(3));
394 reg_def XMM9e( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(4));
395 reg_def XMM9f( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(5));
396 reg_def XMM9g( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(6));
397 reg_def XMM9h( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(7));
398 reg_def XMM9i( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(8));
399 reg_def XMM9j( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(9));
400 reg_def XMM9k( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(10));
401 reg_def XMM9l( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(11));
402 reg_def XMM9m( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(12));
403 reg_def XMM9n( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(13));
404 reg_def XMM9o( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(14));
405 reg_def XMM9p( SOC, SOC, Op_RegF, 9, xmm9->as_VMReg()->next(15));
406
407 reg_def XMM10 ( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg());
408 reg_def XMM10b( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(1));
409 reg_def XMM10c( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(2));
410 reg_def XMM10d( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(3));
411 reg_def XMM10e( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(4));
412 reg_def XMM10f( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(5));
413 reg_def XMM10g( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(6));
414 reg_def XMM10h( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(7));
415 reg_def XMM10i( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(8));
416 reg_def XMM10j( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(9));
417 reg_def XMM10k( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(10));
418 reg_def XMM10l( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(11));
419 reg_def XMM10m( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(12));
420 reg_def XMM10n( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(13));
421 reg_def XMM10o( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(14));
422 reg_def XMM10p( SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next(15));
423
424 reg_def XMM11 ( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg());
425 reg_def XMM11b( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(1));
426 reg_def XMM11c( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(2));
427 reg_def XMM11d( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(3));
428 reg_def XMM11e( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(4));
429 reg_def XMM11f( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(5));
430 reg_def XMM11g( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(6));
431 reg_def XMM11h( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(7));
432 reg_def XMM11i( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(8));
433 reg_def XMM11j( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(9));
434 reg_def XMM11k( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(10));
435 reg_def XMM11l( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(11));
436 reg_def XMM11m( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(12));
437 reg_def XMM11n( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(13));
438 reg_def XMM11o( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(14));
439 reg_def XMM11p( SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next(15));
440
441 reg_def XMM12 ( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg());
442 reg_def XMM12b( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(1));
443 reg_def XMM12c( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(2));
444 reg_def XMM12d( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(3));
445 reg_def XMM12e( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(4));
446 reg_def XMM12f( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(5));
447 reg_def XMM12g( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(6));
448 reg_def XMM12h( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(7));
449 reg_def XMM12i( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(8));
450 reg_def XMM12j( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(9));
451 reg_def XMM12k( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(10));
452 reg_def XMM12l( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(11));
453 reg_def XMM12m( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(12));
454 reg_def XMM12n( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(13));
455 reg_def XMM12o( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(14));
456 reg_def XMM12p( SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next(15));
457
458 reg_def XMM13 ( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg());
459 reg_def XMM13b( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(1));
460 reg_def XMM13c( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(2));
461 reg_def XMM13d( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(3));
462 reg_def XMM13e( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(4));
463 reg_def XMM13f( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(5));
464 reg_def XMM13g( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(6));
465 reg_def XMM13h( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(7));
466 reg_def XMM13i( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(8));
467 reg_def XMM13j( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(9));
468 reg_def XMM13k( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(10));
469 reg_def XMM13l( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(11));
470 reg_def XMM13m( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(12));
471 reg_def XMM13n( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(13));
472 reg_def XMM13o( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(14));
473 reg_def XMM13p( SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next(15));
474
475 reg_def XMM14 ( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg());
476 reg_def XMM14b( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(1));
477 reg_def XMM14c( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(2));
478 reg_def XMM14d( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(3));
479 reg_def XMM14e( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(4));
480 reg_def XMM14f( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(5));
481 reg_def XMM14g( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(6));
482 reg_def XMM14h( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(7));
483 reg_def XMM14i( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(8));
484 reg_def XMM14j( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(9));
485 reg_def XMM14k( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(10));
486 reg_def XMM14l( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(11));
487 reg_def XMM14m( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(12));
488 reg_def XMM14n( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(13));
489 reg_def XMM14o( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(14));
490 reg_def XMM14p( SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next(15));
491
492 reg_def XMM15 ( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg());
493 reg_def XMM15b( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(1));
494 reg_def XMM15c( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(2));
495 reg_def XMM15d( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(3));
496 reg_def XMM15e( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(4));
497 reg_def XMM15f( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(5));
498 reg_def XMM15g( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(6));
499 reg_def XMM15h( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(7));
500 reg_def XMM15i( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(8));
501 reg_def XMM15j( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(9));
502 reg_def XMM15k( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(10));
503 reg_def XMM15l( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(11));
504 reg_def XMM15m( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(12));
505 reg_def XMM15n( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(13));
506 reg_def XMM15o( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(14));
507 reg_def XMM15p( SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next(15));
508
509 reg_def XMM16 ( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg());
510 reg_def XMM16b( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(1));
511 reg_def XMM16c( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(2));
512 reg_def XMM16d( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(3));
513 reg_def XMM16e( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(4));
514 reg_def XMM16f( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(5));
515 reg_def XMM16g( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(6));
516 reg_def XMM16h( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(7));
517 reg_def XMM16i( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(8));
518 reg_def XMM16j( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(9));
519 reg_def XMM16k( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(10));
520 reg_def XMM16l( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(11));
521 reg_def XMM16m( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(12));
522 reg_def XMM16n( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(13));
523 reg_def XMM16o( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(14));
524 reg_def XMM16p( SOC, SOC, Op_RegF, 16, xmm16->as_VMReg()->next(15));
525
526 reg_def XMM17 ( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg());
527 reg_def XMM17b( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(1));
528 reg_def XMM17c( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(2));
529 reg_def XMM17d( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(3));
530 reg_def XMM17e( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(4));
531 reg_def XMM17f( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(5));
532 reg_def XMM17g( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(6));
533 reg_def XMM17h( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(7));
534 reg_def XMM17i( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(8));
535 reg_def XMM17j( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(9));
536 reg_def XMM17k( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(10));
537 reg_def XMM17l( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(11));
538 reg_def XMM17m( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(12));
539 reg_def XMM17n( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(13));
540 reg_def XMM17o( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(14));
541 reg_def XMM17p( SOC, SOC, Op_RegF, 17, xmm17->as_VMReg()->next(15));
542
543 reg_def XMM18 ( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg());
544 reg_def XMM18b( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(1));
545 reg_def XMM18c( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(2));
546 reg_def XMM18d( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(3));
547 reg_def XMM18e( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(4));
548 reg_def XMM18f( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(5));
549 reg_def XMM18g( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(6));
550 reg_def XMM18h( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(7));
551 reg_def XMM18i( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(8));
552 reg_def XMM18j( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(9));
553 reg_def XMM18k( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(10));
554 reg_def XMM18l( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(11));
555 reg_def XMM18m( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(12));
556 reg_def XMM18n( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(13));
557 reg_def XMM18o( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(14));
558 reg_def XMM18p( SOC, SOC, Op_RegF, 18, xmm18->as_VMReg()->next(15));
559
560 reg_def XMM19 ( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg());
561 reg_def XMM19b( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(1));
562 reg_def XMM19c( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(2));
563 reg_def XMM19d( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(3));
564 reg_def XMM19e( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(4));
565 reg_def XMM19f( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(5));
566 reg_def XMM19g( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(6));
567 reg_def XMM19h( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(7));
568 reg_def XMM19i( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(8));
569 reg_def XMM19j( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(9));
570 reg_def XMM19k( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(10));
571 reg_def XMM19l( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(11));
572 reg_def XMM19m( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(12));
573 reg_def XMM19n( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(13));
574 reg_def XMM19o( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(14));
575 reg_def XMM19p( SOC, SOC, Op_RegF, 19, xmm19->as_VMReg()->next(15));
576
577 reg_def XMM20 ( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg());
578 reg_def XMM20b( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(1));
579 reg_def XMM20c( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(2));
580 reg_def XMM20d( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(3));
581 reg_def XMM20e( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(4));
582 reg_def XMM20f( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(5));
583 reg_def XMM20g( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(6));
584 reg_def XMM20h( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(7));
585 reg_def XMM20i( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(8));
586 reg_def XMM20j( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(9));
587 reg_def XMM20k( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(10));
588 reg_def XMM20l( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(11));
589 reg_def XMM20m( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(12));
590 reg_def XMM20n( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(13));
591 reg_def XMM20o( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(14));
592 reg_def XMM20p( SOC, SOC, Op_RegF, 20, xmm20->as_VMReg()->next(15));
593
594 reg_def XMM21 ( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg());
595 reg_def XMM21b( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(1));
596 reg_def XMM21c( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(2));
597 reg_def XMM21d( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(3));
598 reg_def XMM21e( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(4));
599 reg_def XMM21f( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(5));
600 reg_def XMM21g( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(6));
601 reg_def XMM21h( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(7));
602 reg_def XMM21i( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(8));
603 reg_def XMM21j( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(9));
604 reg_def XMM21k( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(10));
605 reg_def XMM21l( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(11));
606 reg_def XMM21m( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(12));
607 reg_def XMM21n( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(13));
608 reg_def XMM21o( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(14));
609 reg_def XMM21p( SOC, SOC, Op_RegF, 21, xmm21->as_VMReg()->next(15));
610
611 reg_def XMM22 ( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg());
612 reg_def XMM22b( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(1));
613 reg_def XMM22c( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(2));
614 reg_def XMM22d( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(3));
615 reg_def XMM22e( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(4));
616 reg_def XMM22f( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(5));
617 reg_def XMM22g( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(6));
618 reg_def XMM22h( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(7));
619 reg_def XMM22i( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(8));
620 reg_def XMM22j( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(9));
621 reg_def XMM22k( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(10));
622 reg_def XMM22l( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(11));
623 reg_def XMM22m( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(12));
624 reg_def XMM22n( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(13));
625 reg_def XMM22o( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(14));
626 reg_def XMM22p( SOC, SOC, Op_RegF, 22, xmm22->as_VMReg()->next(15));
627
628 reg_def XMM23 ( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg());
629 reg_def XMM23b( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(1));
630 reg_def XMM23c( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(2));
631 reg_def XMM23d( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(3));
632 reg_def XMM23e( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(4));
633 reg_def XMM23f( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(5));
634 reg_def XMM23g( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(6));
635 reg_def XMM23h( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(7));
636 reg_def XMM23i( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(8));
637 reg_def XMM23j( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(9));
638 reg_def XMM23k( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(10));
639 reg_def XMM23l( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(11));
640 reg_def XMM23m( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(12));
641 reg_def XMM23n( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(13));
642 reg_def XMM23o( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(14));
643 reg_def XMM23p( SOC, SOC, Op_RegF, 23, xmm23->as_VMReg()->next(15));
644
645 reg_def XMM24 ( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg());
646 reg_def XMM24b( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(1));
647 reg_def XMM24c( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(2));
648 reg_def XMM24d( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(3));
649 reg_def XMM24e( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(4));
650 reg_def XMM24f( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(5));
651 reg_def XMM24g( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(6));
652 reg_def XMM24h( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(7));
653 reg_def XMM24i( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(8));
654 reg_def XMM24j( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(9));
655 reg_def XMM24k( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(10));
656 reg_def XMM24l( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(11));
657 reg_def XMM24m( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(12));
658 reg_def XMM24n( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(13));
659 reg_def XMM24o( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(14));
660 reg_def XMM24p( SOC, SOC, Op_RegF, 24, xmm24->as_VMReg()->next(15));
661
662 reg_def XMM25 ( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg());
663 reg_def XMM25b( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(1));
664 reg_def XMM25c( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(2));
665 reg_def XMM25d( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(3));
666 reg_def XMM25e( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(4));
667 reg_def XMM25f( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(5));
668 reg_def XMM25g( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(6));
669 reg_def XMM25h( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(7));
670 reg_def XMM25i( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(8));
671 reg_def XMM25j( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(9));
672 reg_def XMM25k( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(10));
673 reg_def XMM25l( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(11));
674 reg_def XMM25m( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(12));
675 reg_def XMM25n( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(13));
676 reg_def XMM25o( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(14));
677 reg_def XMM25p( SOC, SOC, Op_RegF, 25, xmm25->as_VMReg()->next(15));
678
679 reg_def XMM26 ( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg());
680 reg_def XMM26b( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(1));
681 reg_def XMM26c( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(2));
682 reg_def XMM26d( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(3));
683 reg_def XMM26e( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(4));
684 reg_def XMM26f( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(5));
685 reg_def XMM26g( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(6));
686 reg_def XMM26h( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(7));
687 reg_def XMM26i( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(8));
688 reg_def XMM26j( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(9));
689 reg_def XMM26k( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(10));
690 reg_def XMM26l( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(11));
691 reg_def XMM26m( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(12));
692 reg_def XMM26n( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(13));
693 reg_def XMM26o( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(14));
694 reg_def XMM26p( SOC, SOC, Op_RegF, 26, xmm26->as_VMReg()->next(15));
695
696 reg_def XMM27 ( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg());
697 reg_def XMM27b( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(1));
698 reg_def XMM27c( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(2));
699 reg_def XMM27d( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(3));
700 reg_def XMM27e( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(4));
701 reg_def XMM27f( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(5));
702 reg_def XMM27g( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(6));
703 reg_def XMM27h( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(7));
704 reg_def XMM27i( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(8));
705 reg_def XMM27j( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(9));
706 reg_def XMM27k( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(10));
707 reg_def XMM27l( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(11));
708 reg_def XMM27m( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(12));
709 reg_def XMM27n( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(13));
710 reg_def XMM27o( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(14));
711 reg_def XMM27p( SOC, SOC, Op_RegF, 27, xmm27->as_VMReg()->next(15));
712
713 reg_def XMM28 ( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg());
714 reg_def XMM28b( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(1));
715 reg_def XMM28c( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(2));
716 reg_def XMM28d( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(3));
717 reg_def XMM28e( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(4));
718 reg_def XMM28f( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(5));
719 reg_def XMM28g( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(6));
720 reg_def XMM28h( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(7));
721 reg_def XMM28i( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(8));
722 reg_def XMM28j( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(9));
723 reg_def XMM28k( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(10));
724 reg_def XMM28l( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(11));
725 reg_def XMM28m( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(12));
726 reg_def XMM28n( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(13));
727 reg_def XMM28o( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(14));
728 reg_def XMM28p( SOC, SOC, Op_RegF, 28, xmm28->as_VMReg()->next(15));
729
730 reg_def XMM29 ( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg());
731 reg_def XMM29b( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(1));
732 reg_def XMM29c( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(2));
733 reg_def XMM29d( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(3));
734 reg_def XMM29e( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(4));
735 reg_def XMM29f( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(5));
736 reg_def XMM29g( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(6));
737 reg_def XMM29h( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(7));
738 reg_def XMM29i( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(8));
739 reg_def XMM29j( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(9));
740 reg_def XMM29k( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(10));
741 reg_def XMM29l( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(11));
742 reg_def XMM29m( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(12));
743 reg_def XMM29n( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(13));
744 reg_def XMM29o( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(14));
745 reg_def XMM29p( SOC, SOC, Op_RegF, 29, xmm29->as_VMReg()->next(15));
746
747 reg_def XMM30 ( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg());
748 reg_def XMM30b( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(1));
749 reg_def XMM30c( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(2));
750 reg_def XMM30d( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(3));
751 reg_def XMM30e( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(4));
752 reg_def XMM30f( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(5));
753 reg_def XMM30g( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(6));
754 reg_def XMM30h( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(7));
755 reg_def XMM30i( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(8));
756 reg_def XMM30j( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(9));
757 reg_def XMM30k( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(10));
758 reg_def XMM30l( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(11));
759 reg_def XMM30m( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(12));
760 reg_def XMM30n( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(13));
761 reg_def XMM30o( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(14));
762 reg_def XMM30p( SOC, SOC, Op_RegF, 30, xmm30->as_VMReg()->next(15));
763
764 reg_def XMM31 ( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg());
765 reg_def XMM31b( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(1));
766 reg_def XMM31c( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(2));
767 reg_def XMM31d( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(3));
768 reg_def XMM31e( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(4));
769 reg_def XMM31f( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(5));
770 reg_def XMM31g( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(6));
771 reg_def XMM31h( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(7));
772 reg_def XMM31i( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(8));
773 reg_def XMM31j( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(9));
774 reg_def XMM31k( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(10));
775 reg_def XMM31l( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(11));
776 reg_def XMM31m( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(12));
777 reg_def XMM31n( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(13));
778 reg_def XMM31o( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(14));
779 reg_def XMM31p( SOC, SOC, Op_RegF, 31, xmm31->as_VMReg()->next(15));
780
781 reg_def RFLAGS(SOC, SOC, 0, 16, VMRegImpl::Bad());
782
783 // AVX3 Mask Registers.
784 reg_def K1 (SOC, SOC, Op_RegI, 1, k1->as_VMReg());
785 reg_def K1_H (SOC, SOC, Op_RegI, 1, k1->as_VMReg()->next());
786
787 reg_def K2 (SOC, SOC, Op_RegI, 2, k2->as_VMReg());
788 reg_def K2_H (SOC, SOC, Op_RegI, 2, k2->as_VMReg()->next());
789
790 reg_def K3 (SOC, SOC, Op_RegI, 3, k3->as_VMReg());
791 reg_def K3_H (SOC, SOC, Op_RegI, 3, k3->as_VMReg()->next());
792
793 reg_def K4 (SOC, SOC, Op_RegI, 4, k4->as_VMReg());
794 reg_def K4_H (SOC, SOC, Op_RegI, 4, k4->as_VMReg()->next());
795
796 reg_def K5 (SOC, SOC, Op_RegI, 5, k5->as_VMReg());
797 reg_def K5_H (SOC, SOC, Op_RegI, 5, k5->as_VMReg()->next());
798
799 reg_def K6 (SOC, SOC, Op_RegI, 6, k6->as_VMReg());
800 reg_def K6_H (SOC, SOC, Op_RegI, 6, k6->as_VMReg()->next());
801
802 reg_def K7 (SOC, SOC, Op_RegI, 7, k7->as_VMReg());
803 reg_def K7_H (SOC, SOC, Op_RegI, 7, k7->as_VMReg()->next());
804
805
806 //----------Architecture Description Register Classes--------------------------
807 // Several register classes are automatically defined based upon information in
808 // this architecture description.
809 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
810 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
811 //
812
813 // Empty register class.
814 reg_class no_reg();
815
816 // Class for all pointer/long registers including APX extended GPRs.
817 reg_class all_reg(RAX, RAX_H,
818 RDX, RDX_H,
819 RBP, RBP_H,
820 RDI, RDI_H,
821 RSI, RSI_H,
822 RCX, RCX_H,
823 RBX, RBX_H,
824 RSP, RSP_H,
825 R8, R8_H,
826 R9, R9_H,
827 R10, R10_H,
828 R11, R11_H,
829 R12, R12_H,
830 R13, R13_H,
831 R14, R14_H,
832 R15, R15_H,
833 R16, R16_H,
834 R17, R17_H,
835 R18, R18_H,
836 R19, R19_H,
837 R20, R20_H,
838 R21, R21_H,
839 R22, R22_H,
840 R23, R23_H,
841 R24, R24_H,
842 R25, R25_H,
843 R26, R26_H,
844 R27, R27_H,
845 R28, R28_H,
846 R29, R29_H,
847 R30, R30_H,
848 R31, R31_H);
849
850 // Class for all int registers including APX extended GPRs.
851 reg_class all_int_reg(RAX
852 RDX,
853 RBP,
854 RDI,
855 RSI,
856 RCX,
857 RBX,
858 R8,
859 R9,
860 R10,
861 R11,
862 R12,
863 R13,
864 R14,
865 R16,
866 R17,
867 R18,
868 R19,
869 R20,
870 R21,
871 R22,
872 R23,
873 R24,
874 R25,
875 R26,
876 R27,
877 R28,
878 R29,
879 R30,
880 R31);
881
882 // Class for all pointer registers
883 reg_class any_reg %{
884 return _ANY_REG_mask;
885 %}
886
887 // Class for all pointer registers (excluding RSP)
888 reg_class ptr_reg %{
889 return _PTR_REG_mask;
890 %}
891
892 // Class for all pointer registers (excluding RSP and RBP)
893 reg_class ptr_reg_no_rbp %{
894 return _PTR_REG_NO_RBP_mask;
895 %}
896
897 // Class for all pointer registers (excluding RAX and RSP)
898 reg_class ptr_no_rax_reg %{
899 return _PTR_NO_RAX_REG_mask;
900 %}
901
902 // Class for all pointer registers (excluding RAX, RBX, and RSP)
903 reg_class ptr_no_rax_rbx_reg %{
904 return _PTR_NO_RAX_RBX_REG_mask;
905 %}
906
907 // Class for all long registers (excluding RSP)
908 reg_class long_reg %{
909 return _LONG_REG_mask;
910 %}
911
912 // Class for all long registers (excluding RAX, RDX and RSP)
913 reg_class long_no_rax_rdx_reg %{
914 return _LONG_NO_RAX_RDX_REG_mask;
915 %}
916
917 // Class for all long registers (excluding RCX and RSP)
918 reg_class long_no_rcx_reg %{
919 return _LONG_NO_RCX_REG_mask;
920 %}
921
922 // Class for all long registers (excluding RBP and R13)
923 reg_class long_no_rbp_r13_reg %{
924 return _LONG_NO_RBP_R13_REG_mask;
925 %}
926
927 // Class for all int registers (excluding RSP)
928 reg_class int_reg %{
929 return _INT_REG_mask;
930 %}
931
932 // Class for all int registers (excluding RAX, RDX, and RSP)
933 reg_class int_no_rax_rdx_reg %{
934 return _INT_NO_RAX_RDX_REG_mask;
935 %}
936
937 // Class for all int registers (excluding RCX and RSP)
938 reg_class int_no_rcx_reg %{
939 return _INT_NO_RCX_REG_mask;
940 %}
941
942 // Class for all int registers (excluding RBP and R13)
943 reg_class int_no_rbp_r13_reg %{
944 return _INT_NO_RBP_R13_REG_mask;
945 %}
946
947 // Singleton class for RAX pointer register
948 reg_class ptr_rax_reg(RAX, RAX_H);
949
950 // Singleton class for RBX pointer register
951 reg_class ptr_rbx_reg(RBX, RBX_H);
952
953 // Singleton class for RSI pointer register
954 reg_class ptr_rsi_reg(RSI, RSI_H);
955
956 // Singleton class for RBP pointer register
957 reg_class ptr_rbp_reg(RBP, RBP_H);
958
959 // Singleton class for RDI pointer register
960 reg_class ptr_rdi_reg(RDI, RDI_H);
961
962 // Singleton class for stack pointer
963 reg_class ptr_rsp_reg(RSP, RSP_H);
964
965 // Singleton class for TLS pointer
966 reg_class ptr_r15_reg(R15, R15_H);
967
968 // Singleton class for RAX long register
969 reg_class long_rax_reg(RAX, RAX_H);
970
971 // Singleton class for RCX long register
972 reg_class long_rcx_reg(RCX, RCX_H);
973
974 // Singleton class for RDX long register
975 reg_class long_rdx_reg(RDX, RDX_H);
976
977 // Singleton class for R11 long register
978 reg_class long_r11_reg(R11, R11_H);
979
980 // Singleton class for RAX int register
981 reg_class int_rax_reg(RAX);
982
983 // Singleton class for RBX int register
984 reg_class int_rbx_reg(RBX);
985
986 // Singleton class for RCX int register
987 reg_class int_rcx_reg(RCX);
988
989 // Singleton class for RDX int register
990 reg_class int_rdx_reg(RDX);
991
992 // Singleton class for RDI int register
993 reg_class int_rdi_reg(RDI);
994
995 // Singleton class for instruction pointer
996 // reg_class ip_reg(RIP);
997
998 alloc_class chunk1(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p,
999 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p,
1000 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p,
1001 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p,
1002 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p,
1003 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p,
1004 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p,
1005 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p,
1006 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p,
1007 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p,
1008 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
1009 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
1010 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
1011 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
1012 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
1013 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p,
1014 XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p,
1015 XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p,
1016 XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p,
1017 XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p,
1018 XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p,
1019 XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p,
1020 XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p,
1021 XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p,
1022 XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p,
1023 XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p,
1024 XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p,
1025 XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p,
1026 XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p,
1027 XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p,
1028 XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p,
1029 XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p);
1030
1031 alloc_class chunk2(K7, K7_H,
1032 K6, K6_H,
1033 K5, K5_H,
1034 K4, K4_H,
1035 K3, K3_H,
1036 K2, K2_H,
1037 K1, K1_H);
1038
1039 reg_class vectmask_reg(K1, K1_H,
1040 K2, K2_H,
1041 K3, K3_H,
1042 K4, K4_H,
1043 K5, K5_H,
1044 K6, K6_H,
1045 K7, K7_H);
1046
1047 reg_class vectmask_reg_K1(K1, K1_H);
1048 reg_class vectmask_reg_K2(K2, K2_H);
1049 reg_class vectmask_reg_K3(K3, K3_H);
1050 reg_class vectmask_reg_K4(K4, K4_H);
1051 reg_class vectmask_reg_K5(K5, K5_H);
1052 reg_class vectmask_reg_K6(K6, K6_H);
1053 reg_class vectmask_reg_K7(K7, K7_H);
1054
1055 // flags allocation class should be last.
1056 alloc_class chunk3(RFLAGS);
1057
1058 // Singleton class for condition codes
1059 reg_class int_flags(RFLAGS);
1060
1061 // Class for pre evex float registers
1062 reg_class float_reg_legacy(XMM0,
1063 XMM1,
1064 XMM2,
1065 XMM3,
1066 XMM4,
1067 XMM5,
1068 XMM6,
1069 XMM7,
1070 XMM8,
1071 XMM9,
1072 XMM10,
1073 XMM11,
1074 XMM12,
1075 XMM13,
1076 XMM14,
1077 XMM15);
1078
1079 // Class for evex float registers
1080 reg_class float_reg_evex(XMM0,
1081 XMM1,
1082 XMM2,
1083 XMM3,
1084 XMM4,
1085 XMM5,
1086 XMM6,
1087 XMM7,
1088 XMM8,
1089 XMM9,
1090 XMM10,
1091 XMM11,
1092 XMM12,
1093 XMM13,
1094 XMM14,
1095 XMM15,
1096 XMM16,
1097 XMM17,
1098 XMM18,
1099 XMM19,
1100 XMM20,
1101 XMM21,
1102 XMM22,
1103 XMM23,
1104 XMM24,
1105 XMM25,
1106 XMM26,
1107 XMM27,
1108 XMM28,
1109 XMM29,
1110 XMM30,
1111 XMM31);
1112
1113 reg_class_dynamic float_reg(float_reg_evex, float_reg_legacy, %{ VM_Version::supports_evex() %} );
1114 reg_class_dynamic float_reg_vl(float_reg_evex, float_reg_legacy, %{ VM_Version::supports_evex() && VM_Version::supports_avx512vl() %} );
1115
1116 // Class for pre evex double registers
1117 reg_class double_reg_legacy(XMM0, XMM0b,
1118 XMM1, XMM1b,
1119 XMM2, XMM2b,
1120 XMM3, XMM3b,
1121 XMM4, XMM4b,
1122 XMM5, XMM5b,
1123 XMM6, XMM6b,
1124 XMM7, XMM7b,
1125 XMM8, XMM8b,
1126 XMM9, XMM9b,
1127 XMM10, XMM10b,
1128 XMM11, XMM11b,
1129 XMM12, XMM12b,
1130 XMM13, XMM13b,
1131 XMM14, XMM14b,
1132 XMM15, XMM15b);
1133
1134 // Class for evex double registers
1135 reg_class double_reg_evex(XMM0, XMM0b,
1136 XMM1, XMM1b,
1137 XMM2, XMM2b,
1138 XMM3, XMM3b,
1139 XMM4, XMM4b,
1140 XMM5, XMM5b,
1141 XMM6, XMM6b,
1142 XMM7, XMM7b,
1143 XMM8, XMM8b,
1144 XMM9, XMM9b,
1145 XMM10, XMM10b,
1146 XMM11, XMM11b,
1147 XMM12, XMM12b,
1148 XMM13, XMM13b,
1149 XMM14, XMM14b,
1150 XMM15, XMM15b,
1151 XMM16, XMM16b,
1152 XMM17, XMM17b,
1153 XMM18, XMM18b,
1154 XMM19, XMM19b,
1155 XMM20, XMM20b,
1156 XMM21, XMM21b,
1157 XMM22, XMM22b,
1158 XMM23, XMM23b,
1159 XMM24, XMM24b,
1160 XMM25, XMM25b,
1161 XMM26, XMM26b,
1162 XMM27, XMM27b,
1163 XMM28, XMM28b,
1164 XMM29, XMM29b,
1165 XMM30, XMM30b,
1166 XMM31, XMM31b);
1167
1168 reg_class_dynamic double_reg(double_reg_evex, double_reg_legacy, %{ VM_Version::supports_evex() %} );
1169 reg_class_dynamic double_reg_vl(double_reg_evex, double_reg_legacy, %{ VM_Version::supports_evex() && VM_Version::supports_avx512vl() %} );
1170
1171 // Class for pre evex 32bit vector registers
1172 reg_class vectors_reg_legacy(XMM0,
1173 XMM1,
1174 XMM2,
1175 XMM3,
1176 XMM4,
1177 XMM5,
1178 XMM6,
1179 XMM7,
1180 XMM8,
1181 XMM9,
1182 XMM10,
1183 XMM11,
1184 XMM12,
1185 XMM13,
1186 XMM14,
1187 XMM15);
1188
1189 // Class for evex 32bit vector registers
1190 reg_class vectors_reg_evex(XMM0,
1191 XMM1,
1192 XMM2,
1193 XMM3,
1194 XMM4,
1195 XMM5,
1196 XMM6,
1197 XMM7,
1198 XMM8,
1199 XMM9,
1200 XMM10,
1201 XMM11,
1202 XMM12,
1203 XMM13,
1204 XMM14,
1205 XMM15,
1206 XMM16,
1207 XMM17,
1208 XMM18,
1209 XMM19,
1210 XMM20,
1211 XMM21,
1212 XMM22,
1213 XMM23,
1214 XMM24,
1215 XMM25,
1216 XMM26,
1217 XMM27,
1218 XMM28,
1219 XMM29,
1220 XMM30,
1221 XMM31);
1222
1223 reg_class_dynamic vectors_reg(vectors_reg_evex, vectors_reg_legacy, %{ VM_Version::supports_evex() %} );
1224 reg_class_dynamic vectors_reg_vlbwdq(vectors_reg_evex, vectors_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1225
1226 // Class for all 64bit vector registers
1227 reg_class vectord_reg_legacy(XMM0, XMM0b,
1228 XMM1, XMM1b,
1229 XMM2, XMM2b,
1230 XMM3, XMM3b,
1231 XMM4, XMM4b,
1232 XMM5, XMM5b,
1233 XMM6, XMM6b,
1234 XMM7, XMM7b,
1235 XMM8, XMM8b,
1236 XMM9, XMM9b,
1237 XMM10, XMM10b,
1238 XMM11, XMM11b,
1239 XMM12, XMM12b,
1240 XMM13, XMM13b,
1241 XMM14, XMM14b,
1242 XMM15, XMM15b);
1243
1244 // Class for all 64bit vector registers
1245 reg_class vectord_reg_evex(XMM0, XMM0b,
1246 XMM1, XMM1b,
1247 XMM2, XMM2b,
1248 XMM3, XMM3b,
1249 XMM4, XMM4b,
1250 XMM5, XMM5b,
1251 XMM6, XMM6b,
1252 XMM7, XMM7b,
1253 XMM8, XMM8b,
1254 XMM9, XMM9b,
1255 XMM10, XMM10b,
1256 XMM11, XMM11b,
1257 XMM12, XMM12b,
1258 XMM13, XMM13b,
1259 XMM14, XMM14b,
1260 XMM15, XMM15b,
1261 XMM16, XMM16b,
1262 XMM17, XMM17b,
1263 XMM18, XMM18b,
1264 XMM19, XMM19b,
1265 XMM20, XMM20b,
1266 XMM21, XMM21b,
1267 XMM22, XMM22b,
1268 XMM23, XMM23b,
1269 XMM24, XMM24b,
1270 XMM25, XMM25b,
1271 XMM26, XMM26b,
1272 XMM27, XMM27b,
1273 XMM28, XMM28b,
1274 XMM29, XMM29b,
1275 XMM30, XMM30b,
1276 XMM31, XMM31b);
1277
1278 reg_class_dynamic vectord_reg(vectord_reg_evex, vectord_reg_legacy, %{ VM_Version::supports_evex() %} );
1279 reg_class_dynamic vectord_reg_vlbwdq(vectord_reg_evex, vectord_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1280
1281 // Class for all 128bit vector registers
1282 reg_class vectorx_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d,
1283 XMM1, XMM1b, XMM1c, XMM1d,
1284 XMM2, XMM2b, XMM2c, XMM2d,
1285 XMM3, XMM3b, XMM3c, XMM3d,
1286 XMM4, XMM4b, XMM4c, XMM4d,
1287 XMM5, XMM5b, XMM5c, XMM5d,
1288 XMM6, XMM6b, XMM6c, XMM6d,
1289 XMM7, XMM7b, XMM7c, XMM7d,
1290 XMM8, XMM8b, XMM8c, XMM8d,
1291 XMM9, XMM9b, XMM9c, XMM9d,
1292 XMM10, XMM10b, XMM10c, XMM10d,
1293 XMM11, XMM11b, XMM11c, XMM11d,
1294 XMM12, XMM12b, XMM12c, XMM12d,
1295 XMM13, XMM13b, XMM13c, XMM13d,
1296 XMM14, XMM14b, XMM14c, XMM14d,
1297 XMM15, XMM15b, XMM15c, XMM15d);
1298
1299 // Class for all 128bit vector registers
1300 reg_class vectorx_reg_evex(XMM0, XMM0b, XMM0c, XMM0d,
1301 XMM1, XMM1b, XMM1c, XMM1d,
1302 XMM2, XMM2b, XMM2c, XMM2d,
1303 XMM3, XMM3b, XMM3c, XMM3d,
1304 XMM4, XMM4b, XMM4c, XMM4d,
1305 XMM5, XMM5b, XMM5c, XMM5d,
1306 XMM6, XMM6b, XMM6c, XMM6d,
1307 XMM7, XMM7b, XMM7c, XMM7d,
1308 XMM8, XMM8b, XMM8c, XMM8d,
1309 XMM9, XMM9b, XMM9c, XMM9d,
1310 XMM10, XMM10b, XMM10c, XMM10d,
1311 XMM11, XMM11b, XMM11c, XMM11d,
1312 XMM12, XMM12b, XMM12c, XMM12d,
1313 XMM13, XMM13b, XMM13c, XMM13d,
1314 XMM14, XMM14b, XMM14c, XMM14d,
1315 XMM15, XMM15b, XMM15c, XMM15d,
1316 XMM16, XMM16b, XMM16c, XMM16d,
1317 XMM17, XMM17b, XMM17c, XMM17d,
1318 XMM18, XMM18b, XMM18c, XMM18d,
1319 XMM19, XMM19b, XMM19c, XMM19d,
1320 XMM20, XMM20b, XMM20c, XMM20d,
1321 XMM21, XMM21b, XMM21c, XMM21d,
1322 XMM22, XMM22b, XMM22c, XMM22d,
1323 XMM23, XMM23b, XMM23c, XMM23d,
1324 XMM24, XMM24b, XMM24c, XMM24d,
1325 XMM25, XMM25b, XMM25c, XMM25d,
1326 XMM26, XMM26b, XMM26c, XMM26d,
1327 XMM27, XMM27b, XMM27c, XMM27d,
1328 XMM28, XMM28b, XMM28c, XMM28d,
1329 XMM29, XMM29b, XMM29c, XMM29d,
1330 XMM30, XMM30b, XMM30c, XMM30d,
1331 XMM31, XMM31b, XMM31c, XMM31d);
1332
1333 reg_class_dynamic vectorx_reg(vectorx_reg_evex, vectorx_reg_legacy, %{ VM_Version::supports_evex() %} );
1334 reg_class_dynamic vectorx_reg_vlbwdq(vectorx_reg_evex, vectorx_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1335
1336 // Class for all 256bit vector registers
1337 reg_class vectory_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h,
1338 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h,
1339 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h,
1340 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h,
1341 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h,
1342 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h,
1343 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h,
1344 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h,
1345 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h,
1346 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h,
1347 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h,
1348 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h,
1349 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h,
1350 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h,
1351 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h,
1352 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h);
1353
1354 // Class for all 256bit vector registers
1355 reg_class vectory_reg_evex(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h,
1356 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h,
1357 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h,
1358 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h,
1359 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h,
1360 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h,
1361 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h,
1362 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h,
1363 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h,
1364 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h,
1365 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h,
1366 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h,
1367 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h,
1368 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h,
1369 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h,
1370 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h,
1371 XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h,
1372 XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h,
1373 XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h,
1374 XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h,
1375 XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h,
1376 XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h,
1377 XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h,
1378 XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h,
1379 XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h,
1380 XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h,
1381 XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h,
1382 XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h,
1383 XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h,
1384 XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h,
1385 XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h,
1386 XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h);
1387
1388 reg_class_dynamic vectory_reg(vectory_reg_evex, vectory_reg_legacy, %{ VM_Version::supports_evex() %} );
1389 reg_class_dynamic vectory_reg_vlbwdq(vectory_reg_evex, vectory_reg_legacy, %{ VM_Version::supports_avx512vlbwdq() %} );
1390
1391 // Class for all 512bit vector registers
1392 reg_class vectorz_reg_evex(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p,
1393 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p,
1394 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p,
1395 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p,
1396 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p,
1397 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p,
1398 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p,
1399 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p,
1400 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p,
1401 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p,
1402 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
1403 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
1404 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
1405 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
1406 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
1407 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p,
1408 XMM16, XMM16b, XMM16c, XMM16d, XMM16e, XMM16f, XMM16g, XMM16h, XMM16i, XMM16j, XMM16k, XMM16l, XMM16m, XMM16n, XMM16o, XMM16p,
1409 XMM17, XMM17b, XMM17c, XMM17d, XMM17e, XMM17f, XMM17g, XMM17h, XMM17i, XMM17j, XMM17k, XMM17l, XMM17m, XMM17n, XMM17o, XMM17p,
1410 XMM18, XMM18b, XMM18c, XMM18d, XMM18e, XMM18f, XMM18g, XMM18h, XMM18i, XMM18j, XMM18k, XMM18l, XMM18m, XMM18n, XMM18o, XMM18p,
1411 XMM19, XMM19b, XMM19c, XMM19d, XMM19e, XMM19f, XMM19g, XMM19h, XMM19i, XMM19j, XMM19k, XMM19l, XMM19m, XMM19n, XMM19o, XMM19p,
1412 XMM20, XMM20b, XMM20c, XMM20d, XMM20e, XMM20f, XMM20g, XMM20h, XMM20i, XMM20j, XMM20k, XMM20l, XMM20m, XMM20n, XMM20o, XMM20p,
1413 XMM21, XMM21b, XMM21c, XMM21d, XMM21e, XMM21f, XMM21g, XMM21h, XMM21i, XMM21j, XMM21k, XMM21l, XMM21m, XMM21n, XMM21o, XMM21p,
1414 XMM22, XMM22b, XMM22c, XMM22d, XMM22e, XMM22f, XMM22g, XMM22h, XMM22i, XMM22j, XMM22k, XMM22l, XMM22m, XMM22n, XMM22o, XMM22p,
1415 XMM23, XMM23b, XMM23c, XMM23d, XMM23e, XMM23f, XMM23g, XMM23h, XMM23i, XMM23j, XMM23k, XMM23l, XMM23m, XMM23n, XMM23o, XMM23p,
1416 XMM24, XMM24b, XMM24c, XMM24d, XMM24e, XMM24f, XMM24g, XMM24h, XMM24i, XMM24j, XMM24k, XMM24l, XMM24m, XMM24n, XMM24o, XMM24p,
1417 XMM25, XMM25b, XMM25c, XMM25d, XMM25e, XMM25f, XMM25g, XMM25h, XMM25i, XMM25j, XMM25k, XMM25l, XMM25m, XMM25n, XMM25o, XMM25p,
1418 XMM26, XMM26b, XMM26c, XMM26d, XMM26e, XMM26f, XMM26g, XMM26h, XMM26i, XMM26j, XMM26k, XMM26l, XMM26m, XMM26n, XMM26o, XMM26p,
1419 XMM27, XMM27b, XMM27c, XMM27d, XMM27e, XMM27f, XMM27g, XMM27h, XMM27i, XMM27j, XMM27k, XMM27l, XMM27m, XMM27n, XMM27o, XMM27p,
1420 XMM28, XMM28b, XMM28c, XMM28d, XMM28e, XMM28f, XMM28g, XMM28h, XMM28i, XMM28j, XMM28k, XMM28l, XMM28m, XMM28n, XMM28o, XMM28p,
1421 XMM29, XMM29b, XMM29c, XMM29d, XMM29e, XMM29f, XMM29g, XMM29h, XMM29i, XMM29j, XMM29k, XMM29l, XMM29m, XMM29n, XMM29o, XMM29p,
1422 XMM30, XMM30b, XMM30c, XMM30d, XMM30e, XMM30f, XMM30g, XMM30h, XMM30i, XMM30j, XMM30k, XMM30l, XMM30m, XMM30n, XMM30o, XMM30p,
1423 XMM31, XMM31b, XMM31c, XMM31d, XMM31e, XMM31f, XMM31g, XMM31h, XMM31i, XMM31j, XMM31k, XMM31l, XMM31m, XMM31n, XMM31o, XMM31p);
1424
1425 // Class for restricted 512bit vector registers
1426 reg_class vectorz_reg_legacy(XMM0, XMM0b, XMM0c, XMM0d, XMM0e, XMM0f, XMM0g, XMM0h, XMM0i, XMM0j, XMM0k, XMM0l, XMM0m, XMM0n, XMM0o, XMM0p,
1427 XMM1, XMM1b, XMM1c, XMM1d, XMM1e, XMM1f, XMM1g, XMM1h, XMM1i, XMM1j, XMM1k, XMM1l, XMM1m, XMM1n, XMM1o, XMM1p,
1428 XMM2, XMM2b, XMM2c, XMM2d, XMM2e, XMM2f, XMM2g, XMM2h, XMM2i, XMM2j, XMM2k, XMM2l, XMM2m, XMM2n, XMM2o, XMM2p,
1429 XMM3, XMM3b, XMM3c, XMM3d, XMM3e, XMM3f, XMM3g, XMM3h, XMM3i, XMM3j, XMM3k, XMM3l, XMM3m, XMM3n, XMM3o, XMM3p,
1430 XMM4, XMM4b, XMM4c, XMM4d, XMM4e, XMM4f, XMM4g, XMM4h, XMM4i, XMM4j, XMM4k, XMM4l, XMM4m, XMM4n, XMM4o, XMM4p,
1431 XMM5, XMM5b, XMM5c, XMM5d, XMM5e, XMM5f, XMM5g, XMM5h, XMM5i, XMM5j, XMM5k, XMM5l, XMM5m, XMM5n, XMM5o, XMM5p,
1432 XMM6, XMM6b, XMM6c, XMM6d, XMM6e, XMM6f, XMM6g, XMM6h, XMM6i, XMM6j, XMM6k, XMM6l, XMM6m, XMM6n, XMM6o, XMM6p,
1433 XMM7, XMM7b, XMM7c, XMM7d, XMM7e, XMM7f, XMM7g, XMM7h, XMM7i, XMM7j, XMM7k, XMM7l, XMM7m, XMM7n, XMM7o, XMM7p,
1434 XMM8, XMM8b, XMM8c, XMM8d, XMM8e, XMM8f, XMM8g, XMM8h, XMM8i, XMM8j, XMM8k, XMM8l, XMM8m, XMM8n, XMM8o, XMM8p,
1435 XMM9, XMM9b, XMM9c, XMM9d, XMM9e, XMM9f, XMM9g, XMM9h, XMM9i, XMM9j, XMM9k, XMM9l, XMM9m, XMM9n, XMM9o, XMM9p,
1436 XMM10, XMM10b, XMM10c, XMM10d, XMM10e, XMM10f, XMM10g, XMM10h, XMM10i, XMM10j, XMM10k, XMM10l, XMM10m, XMM10n, XMM10o, XMM10p,
1437 XMM11, XMM11b, XMM11c, XMM11d, XMM11e, XMM11f, XMM11g, XMM11h, XMM11i, XMM11j, XMM11k, XMM11l, XMM11m, XMM11n, XMM11o, XMM11p,
1438 XMM12, XMM12b, XMM12c, XMM12d, XMM12e, XMM12f, XMM12g, XMM12h, XMM12i, XMM12j, XMM12k, XMM12l, XMM12m, XMM12n, XMM12o, XMM12p,
1439 XMM13, XMM13b, XMM13c, XMM13d, XMM13e, XMM13f, XMM13g, XMM13h, XMM13i, XMM13j, XMM13k, XMM13l, XMM13m, XMM13n, XMM13o, XMM13p,
1440 XMM14, XMM14b, XMM14c, XMM14d, XMM14e, XMM14f, XMM14g, XMM14h, XMM14i, XMM14j, XMM14k, XMM14l, XMM14m, XMM14n, XMM14o, XMM14p,
1441 XMM15, XMM15b, XMM15c, XMM15d, XMM15e, XMM15f, XMM15g, XMM15h, XMM15i, XMM15j, XMM15k, XMM15l, XMM15m, XMM15n, XMM15o, XMM15p);
1442
1443 reg_class_dynamic vectorz_reg (vectorz_reg_evex, vectorz_reg_legacy, %{ VM_Version::supports_evex() %} );
1444 reg_class_dynamic vectorz_reg_vl(vectorz_reg_evex, vectorz_reg_legacy, %{ VM_Version::supports_evex() && VM_Version::supports_avx512vl() %} );
1445
1446 reg_class xmm0_reg(XMM0, XMM0b, XMM0c, XMM0d);
1447
1448 %}
1449
1450
1451 //----------SOURCE BLOCK-------------------------------------------------------
1452 // This is a block of C++ code which provides values, functions, and
1453 // definitions necessary in the rest of the architecture description
1454
1455 source_hpp %{
1456
1457 #include "peephole_x86_64.hpp"
1458
1459 bool castLL_is_imm32(const Node* n);
1460
1461 %}
1462
1463 source %{
1464
1465 bool castLL_is_imm32(const Node* n) {
1466 assert(n->is_CastLL(), "must be a CastLL");
1467 const TypeLong* t = n->bottom_type()->is_long();
1468 return (t->_lo == min_jlong || Assembler::is_simm32(t->_lo)) && (t->_hi == max_jlong || Assembler::is_simm32(t->_hi));
1469 }
1470
1471 %}
1472
1473 // Register masks
1474 source_hpp %{
1475
1476 extern RegMask _ANY_REG_mask;
1477 extern RegMask _PTR_REG_mask;
1478 extern RegMask _PTR_REG_NO_RBP_mask;
1479 extern RegMask _PTR_NO_RAX_REG_mask;
1480 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
1481 extern RegMask _LONG_REG_mask;
1482 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
1483 extern RegMask _LONG_NO_RCX_REG_mask;
1484 extern RegMask _LONG_NO_RBP_R13_REG_mask;
1485 extern RegMask _INT_REG_mask;
1486 extern RegMask _INT_NO_RAX_RDX_REG_mask;
1487 extern RegMask _INT_NO_RCX_REG_mask;
1488 extern RegMask _INT_NO_RBP_R13_REG_mask;
1489 extern RegMask _FLOAT_REG_mask;
1490
1491 extern RegMask _STACK_OR_PTR_REG_mask;
1492 extern RegMask _STACK_OR_LONG_REG_mask;
1493 extern RegMask _STACK_OR_INT_REG_mask;
1494
1495 inline const RegMask& STACK_OR_PTR_REG_mask() { return _STACK_OR_PTR_REG_mask; }
1496 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
1497 inline const RegMask& STACK_OR_INT_REG_mask() { return _STACK_OR_INT_REG_mask; }
1498
1499 %}
1500
1501 source %{
1502 #define RELOC_IMM64 Assembler::imm_operand
1503 #define RELOC_DISP32 Assembler::disp32_operand
1504
1505 #define __ masm->
1506
1507 RegMask _ANY_REG_mask;
1508 RegMask _PTR_REG_mask;
1509 RegMask _PTR_REG_NO_RBP_mask;
1510 RegMask _PTR_NO_RAX_REG_mask;
1511 RegMask _PTR_NO_RAX_RBX_REG_mask;
1512 RegMask _LONG_REG_mask;
1513 RegMask _LONG_NO_RAX_RDX_REG_mask;
1514 RegMask _LONG_NO_RCX_REG_mask;
1515 RegMask _LONG_NO_RBP_R13_REG_mask;
1516 RegMask _INT_REG_mask;
1517 RegMask _INT_NO_RAX_RDX_REG_mask;
1518 RegMask _INT_NO_RCX_REG_mask;
1519 RegMask _INT_NO_RBP_R13_REG_mask;
1520 RegMask _FLOAT_REG_mask;
1521 RegMask _STACK_OR_PTR_REG_mask;
1522 RegMask _STACK_OR_LONG_REG_mask;
1523 RegMask _STACK_OR_INT_REG_mask;
1524
1525 static bool need_r12_heapbase() {
1526 return UseCompressedOops;
1527 }
1528
1529 void reg_mask_init() {
1530 constexpr Register egprs[] = {r16, r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31};
1531
1532 // _ALL_REG_mask is generated by adlc from the all_reg register class below.
1533 // We derive a number of subsets from it.
1534 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1535
1536 if (PreserveFramePointer) {
1537 _ANY_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1538 _ANY_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
1539 }
1540 if (need_r12_heapbase()) {
1541 _ANY_REG_mask.remove(OptoReg::as_OptoReg(r12->as_VMReg()));
1542 _ANY_REG_mask.remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
1543 }
1544
1545 _PTR_REG_mask.assignFrom(_ANY_REG_mask);
1546 _PTR_REG_mask.remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
1547 _PTR_REG_mask.remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
1548 _PTR_REG_mask.remove(OptoReg::as_OptoReg(r15->as_VMReg()));
1549 _PTR_REG_mask.remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
1550 if (!UseAPX) {
1551 for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
1552 _PTR_REG_mask.remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
1553 _PTR_REG_mask.remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()->next()));
1554 }
1555 }
1556
1557 _STACK_OR_PTR_REG_mask.assignFrom(_PTR_REG_mask);
1558 _STACK_OR_PTR_REG_mask.or_with(STACK_OR_STACK_SLOTS_mask());
1559
1560 _PTR_REG_NO_RBP_mask.assignFrom(_PTR_REG_mask);
1561 _PTR_REG_NO_RBP_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1562 _PTR_REG_NO_RBP_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
1563
1564 _PTR_NO_RAX_REG_mask.assignFrom(_PTR_REG_mask);
1565 _PTR_NO_RAX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()));
1566 _PTR_NO_RAX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
1567
1568 _PTR_NO_RAX_RBX_REG_mask.assignFrom(_PTR_NO_RAX_REG_mask);
1569 _PTR_NO_RAX_RBX_REG_mask.remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
1570 _PTR_NO_RAX_RBX_REG_mask.remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
1571
1572
1573 _LONG_REG_mask.assignFrom(_PTR_REG_mask);
1574 _STACK_OR_LONG_REG_mask.assignFrom(_LONG_REG_mask);
1575 _STACK_OR_LONG_REG_mask.or_with(STACK_OR_STACK_SLOTS_mask());
1576
1577 _LONG_NO_RAX_RDX_REG_mask.assignFrom(_LONG_REG_mask);
1578 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()));
1579 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
1580 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
1581 _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
1582
1583 _LONG_NO_RCX_REG_mask.assignFrom(_LONG_REG_mask);
1584 _LONG_NO_RCX_REG_mask.remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
1585 _LONG_NO_RCX_REG_mask.remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
1586
1587 _LONG_NO_RBP_R13_REG_mask.assignFrom(_LONG_REG_mask);
1588 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1589 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
1590 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(r13->as_VMReg()));
1591 _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
1592
1593 _INT_REG_mask.assignFrom(_ALL_INT_REG_mask);
1594 if (!UseAPX) {
1595 for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
1596 _INT_REG_mask.remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
1597 }
1598 }
1599
1600 if (PreserveFramePointer) {
1601 _INT_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1602 }
1603 if (need_r12_heapbase()) {
1604 _INT_REG_mask.remove(OptoReg::as_OptoReg(r12->as_VMReg()));
1605 }
1606
1607 _STACK_OR_INT_REG_mask.assignFrom(_INT_REG_mask);
1608 _STACK_OR_INT_REG_mask.or_with(STACK_OR_STACK_SLOTS_mask());
1609
1610 _INT_NO_RAX_RDX_REG_mask.assignFrom(_INT_REG_mask);
1611 _INT_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()));
1612 _INT_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
1613
1614 _INT_NO_RCX_REG_mask.assignFrom(_INT_REG_mask);
1615 _INT_NO_RCX_REG_mask.remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
1616
1617 _INT_NO_RBP_R13_REG_mask.assignFrom(_INT_REG_mask);
1618 _INT_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
1619 _INT_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(r13->as_VMReg()));
1620
1621 // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
1622 // from the float_reg_legacy/float_reg_evex register class.
1623 _FLOAT_REG_mask.assignFrom(VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask);
1624 }
1625
1626 static bool generate_vzeroupper(Compile* C) {
1627 return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false; // Generate vzeroupper
1628 }
1629
1630 static int clear_avx_size() {
1631 return generate_vzeroupper(Compile::current()) ? 3: 0; // vzeroupper
1632 }
1633
1634 // !!!!! Special hack to get all types of calls to specify the byte offset
1635 // from the start of the call to the point where the return address
1636 // will point.
1637 int MachCallStaticJavaNode::ret_addr_offset() const
1638 {
1639 int offset = 5; // 5 bytes from start of call to where return address points
1640 offset += clear_avx_size();
1641 return offset;
1642 }
1643
1644 int MachCallDynamicJavaNode::ret_addr_offset() const
1645 {
1646 int offset = 15; // 15 bytes from start of call to where return address points
1647 offset += clear_avx_size();
1648 return offset;
1649 }
1650
1651 int MachCallRuntimeNode::ret_addr_offset() const {
1652 int offset = 13; // movq r10,#addr; callq (r10)
1653 if (this->ideal_Opcode() != Op_CallLeafVector) {
1654 offset += clear_avx_size();
1655 }
1656 return offset;
1657 }
1658 //
1659 // Compute padding required for nodes which need alignment
1660 //
1661
1662 // The address of the call instruction needs to be 4-byte aligned to
1663 // ensure that it does not span a cache line so that it can be patched.
1664 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
1665 {
1666 current_offset += clear_avx_size(); // skip vzeroupper
1667 current_offset += 1; // skip call opcode byte
1668 return align_up(current_offset, alignment_required()) - current_offset;
1669 }
1670
1671 // The address of the call instruction needs to be 4-byte aligned to
1672 // ensure that it does not span a cache line so that it can be patched.
1673 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
1674 {
1675 current_offset += clear_avx_size(); // skip vzeroupper
1676 current_offset += 11; // skip movq instruction + call opcode byte
1677 return align_up(current_offset, alignment_required()) - current_offset;
1678 }
1679
1680 // This could be in MacroAssembler but it's fairly C2 specific
1681 static void emit_cmpfp_fixup(MacroAssembler* masm) {
1682 Label exit;
1683 __ jccb(Assembler::noParity, exit);
1684 __ pushf();
1685 //
1686 // comiss/ucomiss instructions set ZF,PF,CF flags and
1687 // zero OF,AF,SF for NaN values.
1688 // Fixup flags by zeroing ZF,PF so that compare of NaN
1689 // values returns 'less than' result (CF is set).
1690 // Leave the rest of flags unchanged.
1691 //
1692 // 7 6 5 4 3 2 1 0
1693 // |S|Z|r|A|r|P|r|C| (r - reserved bit)
1694 // 0 0 1 0 1 0 1 1 (0x2B)
1695 //
1696 __ andq(Address(rsp, 0), 0xffffff2b);
1697 __ popf();
1698 __ bind(exit);
1699 }
1700
1701 static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
1702 // If any floating point comparison instruction is used, unordered case always triggers jump
1703 // for below condition, CF=1 is true when at least one input is NaN
1704 Label done;
1705 __ movl(dst, -1);
1706 __ jcc(Assembler::below, done);
1707 __ setcc(Assembler::notEqual, dst);
1708 __ bind(done);
1709 }
1710
1711 enum FP_PREC {
1712 fp_prec_hlf,
1713 fp_prec_flt,
1714 fp_prec_dbl
1715 };
1716
1717 static inline void emit_fp_ucom(MacroAssembler* masm, enum FP_PREC pt,
1718 XMMRegister p, XMMRegister q) {
1719 if (pt == fp_prec_hlf) {
1720 __ evucomish(p, q);
1721 } else if (pt == fp_prec_flt) {
1722 __ ucomiss(p, q);
1723 } else {
1724 __ ucomisd(p, q);
1725 }
1726 }
1727
1728 static inline void movfp(MacroAssembler* masm, enum FP_PREC pt,
1729 XMMRegister dst, XMMRegister src, Register scratch) {
1730 if (pt == fp_prec_hlf) {
1731 __ movhlf(dst, src, scratch);
1732 } else if (pt == fp_prec_flt) {
1733 __ movflt(dst, src);
1734 } else {
1735 __ movdbl(dst, src);
1736 }
1737 }
1738
1739 // Math.min() # Math.max()
1740 // -----------------------------
1741 // (v)ucomis[h/s/d] #
1742 // ja -> b # a
1743 // jp -> NaN # NaN
1744 // jb -> a # b
1745 // je -> a | b # a & b
1746 static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
1747 XMMRegister a, XMMRegister b, Register rt,
1748 bool min, enum FP_PREC pt) {
1749 Label nan, zero, below, above, done;
1750
1751 emit_fp_ucom(masm, pt, a, b);
1752
1753 if (dst->encoding() != (min ? b : a)->encoding()) {
1754 __ jccb(Assembler::above, above); // CF=0 & ZF=0
1755 } else {
1756 __ jccb(Assembler::above, done);
1757 }
1758 __ jccb(Assembler::parity, nan); // PF=1
1759 __ jccb(Assembler::below, below); // CF=1
1760
1761 // equal
1762 // Using bitwise operations is a low cost way to compute the correct result
1763 // for zero and non-zero inputs in this scenario except for NaN, which is
1764 // handled separately. The mantissa and exponent are valid with either
1765 // bitwise operation. For zero inputs, the sign bit is chosen according to
1766 // whether a minimum or maximum value is required.
1767 if (min) {
1768 // Negative sign preserved when available (e.g., min(+0, -0) -> -0)
1769 __ vpor(dst, a, b, Assembler::AVX_128bit);
1770 } else {
1771 // Positive sign preserved when available (e.g., max(+0, -0) -> +0)
1772 __ vpand(dst, a, b, Assembler::AVX_128bit);
1773 }
1774 __ jmp(done);
1775
1776 __ bind(above);
1777 movfp(masm, pt, dst, min ? b : a, rt);
1778 __ jmp(done);
1779
1780 __ bind(nan);
1781 if (pt == fp_prec_hlf) {
1782 __ movl(rt, 0x00007e00); // Float16.NaN
1783 __ evmovw(dst, rt);
1784 } else if (pt == fp_prec_flt) {
1785 __ movl(rt, 0x7fc00000); // Float.NaN
1786 __ movdl(dst, rt);
1787 } else {
1788 __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
1789 __ movdq(dst, rt);
1790 }
1791 __ jmp(done);
1792
1793 __ bind(below);
1794 movfp(masm, pt, dst, min ? a : b, rt);
1795
1796 __ bind(done);
1797 }
1798
1799 //=============================================================================
1800 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1801
1802 int ConstantTable::calculate_table_base_offset() const {
1803 return 0; // absolute addressing, no offset
1804 }
1805
1806 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1807 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1808 ShouldNotReachHere();
1809 }
1810
1811 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1812 // Empty encoding
1813 }
1814
1815 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1816 return 0;
1817 }
1818
1819 #ifndef PRODUCT
1820 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1821 st->print("# MachConstantBaseNode (empty encoding)");
1822 }
1823 #endif
1824
1825
1826 //=============================================================================
1827 #ifndef PRODUCT
1828 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1829 Compile* C = ra_->C;
1830
1831 int framesize = C->output()->frame_size_in_bytes();
1832 int bangsize = C->output()->bang_size_in_bytes();
1833 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1834 // Remove wordSize for return addr which is already pushed.
1835 framesize -= wordSize;
1836
1837 if (C->output()->need_stack_bang(bangsize)) {
1838 framesize -= wordSize;
1839 st->print("# stack bang (%d bytes)", bangsize);
1840 st->print("\n\t");
1841 st->print("pushq rbp\t# Save rbp");
1842 if (PreserveFramePointer) {
1843 st->print("\n\t");
1844 st->print("movq rbp, rsp\t# Save the caller's SP into rbp");
1845 }
1846 if (framesize) {
1847 st->print("\n\t");
1848 st->print("subq rsp, #%d\t# Create frame",framesize);
1849 }
1850 } else {
1851 st->print("subq rsp, #%d\t# Create frame",framesize);
1852 st->print("\n\t");
1853 framesize -= wordSize;
1854 st->print("movq [rsp + #%d], rbp\t# Save rbp",framesize);
1855 if (PreserveFramePointer) {
1856 st->print("\n\t");
1857 st->print("movq rbp, rsp\t# Save the caller's SP into rbp");
1858 if (framesize > 0) {
1859 st->print("\n\t");
1860 st->print("addq rbp, #%d", framesize);
1861 }
1862 }
1863 }
1864
1865 if (VerifyStackAtCalls) {
1866 st->print("\n\t");
1867 framesize -= wordSize;
1868 st->print("movq [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
1869 #ifdef ASSERT
1870 st->print("\n\t");
1871 st->print("# stack alignment check");
1872 #endif
1873 }
1874 if (C->stub_function() != nullptr) {
1875 st->print("\n\t");
1876 st->print("cmpl [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
1877 st->print("\n\t");
1878 st->print("je fast_entry\t");
1879 st->print("\n\t");
1880 st->print("call #nmethod_entry_barrier_stub\t");
1881 st->print("\n\tfast_entry:");
1882 }
1883 st->cr();
1884 }
1885 #endif
1886
1887 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1888 Compile* C = ra_->C;
1889
1890 int framesize = C->output()->frame_size_in_bytes();
1891 int bangsize = C->output()->bang_size_in_bytes();
1892
1893 if (C->clinit_barrier_on_entry()) {
1894 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
1895 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1896
1897 Label L_skip_barrier;
1898 Register klass = rscratch1;
1899
1900 __ mov_metadata(klass, C->method()->holder()->constant_encoding());
1901 __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
1902
1903 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
1904
1905 __ bind(L_skip_barrier);
1906 }
1907
1908 __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
1909
1910 C->output()->set_frame_complete(__ offset());
1911
1912 if (C->has_mach_constant_base_node()) {
1913 // NOTE: We set the table base offset here because users might be
1914 // emitted before MachConstantBaseNode.
1915 ConstantTable& constant_table = C->output()->constant_table();
1916 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1917 }
1918 }
1919
1920 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1921 {
1922 return MachNode::size(ra_); // too many variables; just compute it
1923 // the hard way
1924 }
1925
1926 int MachPrologNode::reloc() const
1927 {
1928 return 0; // a large enough number
1929 }
1930
1931 //=============================================================================
1932 #ifndef PRODUCT
1933 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1934 {
1935 Compile* C = ra_->C;
1936 if (generate_vzeroupper(C)) {
1937 st->print("vzeroupper");
1938 st->cr(); st->print("\t");
1939 }
1940
1941 int framesize = C->output()->frame_size_in_bytes();
1942 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1943 // Remove word for return adr already pushed
1944 // and RBP
1945 framesize -= 2*wordSize;
1946
1947 if (framesize) {
1948 st->print_cr("addq rsp, %d\t# Destroy frame", framesize);
1949 st->print("\t");
1950 }
1951
1952 st->print_cr("popq rbp");
1953 if (do_polling() && C->is_method_compilation()) {
1954 st->print("\t");
1955 st->print_cr("cmpq rsp, poll_offset[r15_thread] \n\t"
1956 "ja #safepoint_stub\t"
1957 "# Safepoint: poll for GC");
1958 }
1959 }
1960 #endif
1961
1962 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
1963 {
1964 Compile* C = ra_->C;
1965
1966 if (generate_vzeroupper(C)) {
1967 // Clear upper bits of YMM registers when current compiled code uses
1968 // wide vectors to avoid AVX <-> SSE transition penalty during call.
1969 __ vzeroupper();
1970 }
1971
1972 int framesize = C->output()->frame_size_in_bytes();
1973 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1974 // Remove word for return adr already pushed
1975 // and RBP
1976 framesize -= 2*wordSize;
1977
1978 // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
1979
1980 if (framesize) {
1981 __ addq(rsp, framesize);
1982 }
1983
1984 __ popq(rbp);
1985
1986 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1987 __ reserved_stack_check();
1988 }
1989
1990 if (do_polling() && C->is_method_compilation()) {
1991 Label dummy_label;
1992 Label* code_stub = &dummy_label;
1993 if (!C->output()->in_scratch_emit_size()) {
1994 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1995 C->output()->add_stub(stub);
1996 code_stub = &stub->entry();
1997 }
1998 __ relocate(relocInfo::poll_return_type);
1999 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
2000 }
2001 }
2002
2003 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
2004 {
2005 return MachNode::size(ra_); // too many variables; just compute it
2006 // the hard way
2007 }
2008
2009 int MachEpilogNode::reloc() const
2010 {
2011 return 2; // a large enough number
2012 }
2013
2014 const Pipeline* MachEpilogNode::pipeline() const
2015 {
2016 return MachNode::pipeline_class();
2017 }
2018
2019 //=============================================================================
2020
2021 enum RC {
2022 rc_bad,
2023 rc_int,
2024 rc_kreg,
2025 rc_float,
2026 rc_stack
2027 };
2028
2029 static enum RC rc_class(OptoReg::Name reg)
2030 {
2031 if( !OptoReg::is_valid(reg) ) return rc_bad;
2032
2033 if (OptoReg::is_stack(reg)) return rc_stack;
2034
2035 VMReg r = OptoReg::as_VMReg(reg);
2036
2037 if (r->is_Register()) return rc_int;
2038
2039 if (r->is_KRegister()) return rc_kreg;
2040
2041 assert(r->is_XMMRegister(), "must be");
2042 return rc_float;
2043 }
2044
2045 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
2046 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
2047 int src_hi, int dst_hi, uint ireg, outputStream* st);
2048
2049 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
2050 int stack_offset, int reg, uint ireg, outputStream* st);
2051
2052 static void vec_stack_to_stack_helper(C2_MacroAssembler *masm, int src_offset,
2053 int dst_offset, uint ireg, outputStream* st) {
2054 if (masm) {
2055 switch (ireg) {
2056 case Op_VecS:
2057 __ movq(Address(rsp, -8), rax);
2058 __ movl(rax, Address(rsp, src_offset));
2059 __ movl(Address(rsp, dst_offset), rax);
2060 __ movq(rax, Address(rsp, -8));
2061 break;
2062 case Op_VecD:
2063 __ pushq(Address(rsp, src_offset));
2064 __ popq (Address(rsp, dst_offset));
2065 break;
2066 case Op_VecX:
2067 __ pushq(Address(rsp, src_offset));
2068 __ popq (Address(rsp, dst_offset));
2069 __ pushq(Address(rsp, src_offset+8));
2070 __ popq (Address(rsp, dst_offset+8));
2071 break;
2072 case Op_VecY:
2073 __ vmovdqu(Address(rsp, -32), xmm0);
2074 __ vmovdqu(xmm0, Address(rsp, src_offset));
2075 __ vmovdqu(Address(rsp, dst_offset), xmm0);
2076 __ vmovdqu(xmm0, Address(rsp, -32));
2077 break;
2078 case Op_VecZ:
2079 __ evmovdquq(Address(rsp, -64), xmm0, 2);
2080 __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
2081 __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
2082 __ evmovdquq(xmm0, Address(rsp, -64), 2);
2083 break;
2084 default:
2085 ShouldNotReachHere();
2086 }
2087 #ifndef PRODUCT
2088 } else {
2089 switch (ireg) {
2090 case Op_VecS:
2091 st->print("movq [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
2092 "movl rax, [rsp + #%d]\n\t"
2093 "movl [rsp + #%d], rax\n\t"
2094 "movq rax, [rsp - #8]",
2095 src_offset, dst_offset);
2096 break;
2097 case Op_VecD:
2098 st->print("pushq [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
2099 "popq [rsp + #%d]",
2100 src_offset, dst_offset);
2101 break;
2102 case Op_VecX:
2103 st->print("pushq [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
2104 "popq [rsp + #%d]\n\t"
2105 "pushq [rsp + #%d]\n\t"
2106 "popq [rsp + #%d]",
2107 src_offset, dst_offset, src_offset+8, dst_offset+8);
2108 break;
2109 case Op_VecY:
2110 st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
2111 "vmovdqu xmm0, [rsp + #%d]\n\t"
2112 "vmovdqu [rsp + #%d], xmm0\n\t"
2113 "vmovdqu xmm0, [rsp - #32]",
2114 src_offset, dst_offset);
2115 break;
2116 case Op_VecZ:
2117 st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
2118 "vmovdqu xmm0, [rsp + #%d]\n\t"
2119 "vmovdqu [rsp + #%d], xmm0\n\t"
2120 "vmovdqu xmm0, [rsp - #64]",
2121 src_offset, dst_offset);
2122 break;
2123 default:
2124 ShouldNotReachHere();
2125 }
2126 #endif
2127 }
2128 }
2129
2130 uint MachSpillCopyNode::implementation(C2_MacroAssembler* masm,
2131 PhaseRegAlloc* ra_,
2132 bool do_size,
2133 outputStream* st) const {
2134 assert(masm != nullptr || st != nullptr, "sanity");
2135 // Get registers to move
2136 OptoReg::Name src_second = ra_->get_reg_second(in(1));
2137 OptoReg::Name src_first = ra_->get_reg_first(in(1));
2138 OptoReg::Name dst_second = ra_->get_reg_second(this);
2139 OptoReg::Name dst_first = ra_->get_reg_first(this);
2140
2141 enum RC src_first_rc = rc_class(src_first);
2142 enum RC dst_first_rc = rc_class(dst_first);
2143
2144 assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
2145 "must move at least 1 register" );
2146
2147 if (src_first == dst_first && src_second == dst_second) {
2148 // Self copy, no move
2149 return 0;
2150 }
2151 if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_pvectmask() == nullptr) {
2152 uint ireg = ideal_reg();
2153 assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
2154 assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
2155 if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
2156 // mem -> mem
2157 int src_offset = ra_->reg2offset(src_first);
2158 int dst_offset = ra_->reg2offset(dst_first);
2159 vec_stack_to_stack_helper(masm, src_offset, dst_offset, ireg, st);
2160 } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
2161 vec_mov_helper(masm, src_first, dst_first, src_second, dst_second, ireg, st);
2162 } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
2163 int stack_offset = ra_->reg2offset(dst_first);
2164 vec_spill_helper(masm, false, stack_offset, src_first, ireg, st);
2165 } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
2166 int stack_offset = ra_->reg2offset(src_first);
2167 vec_spill_helper(masm, true, stack_offset, dst_first, ireg, st);
2168 } else {
2169 ShouldNotReachHere();
2170 }
2171 return 0;
2172 }
2173 if (src_first_rc == rc_stack) {
2174 // mem ->
2175 if (dst_first_rc == rc_stack) {
2176 // mem -> mem
2177 assert(src_second != dst_first, "overlap");
2178 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2179 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2180 // 64-bit
2181 int src_offset = ra_->reg2offset(src_first);
2182 int dst_offset = ra_->reg2offset(dst_first);
2183 if (masm) {
2184 __ pushq(Address(rsp, src_offset));
2185 __ popq (Address(rsp, dst_offset));
2186 #ifndef PRODUCT
2187 } else {
2188 st->print("pushq [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
2189 "popq [rsp + #%d]",
2190 src_offset, dst_offset);
2191 #endif
2192 }
2193 } else {
2194 // 32-bit
2195 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2196 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2197 // No pushl/popl, so:
2198 int src_offset = ra_->reg2offset(src_first);
2199 int dst_offset = ra_->reg2offset(dst_first);
2200 if (masm) {
2201 __ movq(Address(rsp, -8), rax);
2202 __ movl(rax, Address(rsp, src_offset));
2203 __ movl(Address(rsp, dst_offset), rax);
2204 __ movq(rax, Address(rsp, -8));
2205 #ifndef PRODUCT
2206 } else {
2207 st->print("movq [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
2208 "movl rax, [rsp + #%d]\n\t"
2209 "movl [rsp + #%d], rax\n\t"
2210 "movq rax, [rsp - #8]",
2211 src_offset, dst_offset);
2212 #endif
2213 }
2214 }
2215 return 0;
2216 } else if (dst_first_rc == rc_int) {
2217 // mem -> gpr
2218 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2219 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2220 // 64-bit
2221 int offset = ra_->reg2offset(src_first);
2222 if (masm) {
2223 __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2224 #ifndef PRODUCT
2225 } else {
2226 st->print("movq %s, [rsp + #%d]\t# spill",
2227 Matcher::regName[dst_first],
2228 offset);
2229 #endif
2230 }
2231 } else {
2232 // 32-bit
2233 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2234 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2235 int offset = ra_->reg2offset(src_first);
2236 if (masm) {
2237 __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2238 #ifndef PRODUCT
2239 } else {
2240 st->print("movl %s, [rsp + #%d]\t# spill",
2241 Matcher::regName[dst_first],
2242 offset);
2243 #endif
2244 }
2245 }
2246 return 0;
2247 } else if (dst_first_rc == rc_float) {
2248 // mem-> xmm
2249 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2250 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2251 // 64-bit
2252 int offset = ra_->reg2offset(src_first);
2253 if (masm) {
2254 __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2255 #ifndef PRODUCT
2256 } else {
2257 st->print("%s %s, [rsp + #%d]\t# spill",
2258 UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
2259 Matcher::regName[dst_first],
2260 offset);
2261 #endif
2262 }
2263 } else {
2264 // 32-bit
2265 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2266 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2267 int offset = ra_->reg2offset(src_first);
2268 if (masm) {
2269 __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2270 #ifndef PRODUCT
2271 } else {
2272 st->print("movss %s, [rsp + #%d]\t# spill",
2273 Matcher::regName[dst_first],
2274 offset);
2275 #endif
2276 }
2277 }
2278 return 0;
2279 } else if (dst_first_rc == rc_kreg) {
2280 // mem -> kreg
2281 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2282 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2283 // 64-bit
2284 int offset = ra_->reg2offset(src_first);
2285 if (masm) {
2286 __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
2287 #ifndef PRODUCT
2288 } else {
2289 st->print("kmovq %s, [rsp + #%d]\t# spill",
2290 Matcher::regName[dst_first],
2291 offset);
2292 #endif
2293 }
2294 }
2295 return 0;
2296 }
2297 } else if (src_first_rc == rc_int) {
2298 // gpr ->
2299 if (dst_first_rc == rc_stack) {
2300 // gpr -> mem
2301 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2302 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2303 // 64-bit
2304 int offset = ra_->reg2offset(dst_first);
2305 if (masm) {
2306 __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
2307 #ifndef PRODUCT
2308 } else {
2309 st->print("movq [rsp + #%d], %s\t# spill",
2310 offset,
2311 Matcher::regName[src_first]);
2312 #endif
2313 }
2314 } else {
2315 // 32-bit
2316 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2317 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2318 int offset = ra_->reg2offset(dst_first);
2319 if (masm) {
2320 __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
2321 #ifndef PRODUCT
2322 } else {
2323 st->print("movl [rsp + #%d], %s\t# spill",
2324 offset,
2325 Matcher::regName[src_first]);
2326 #endif
2327 }
2328 }
2329 return 0;
2330 } else if (dst_first_rc == rc_int) {
2331 // gpr -> gpr
2332 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2333 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2334 // 64-bit
2335 if (masm) {
2336 __ movq(as_Register(Matcher::_regEncode[dst_first]),
2337 as_Register(Matcher::_regEncode[src_first]));
2338 #ifndef PRODUCT
2339 } else {
2340 st->print("movq %s, %s\t# spill",
2341 Matcher::regName[dst_first],
2342 Matcher::regName[src_first]);
2343 #endif
2344 }
2345 return 0;
2346 } else {
2347 // 32-bit
2348 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2349 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2350 if (masm) {
2351 __ movl(as_Register(Matcher::_regEncode[dst_first]),
2352 as_Register(Matcher::_regEncode[src_first]));
2353 #ifndef PRODUCT
2354 } else {
2355 st->print("movl %s, %s\t# spill",
2356 Matcher::regName[dst_first],
2357 Matcher::regName[src_first]);
2358 #endif
2359 }
2360 return 0;
2361 }
2362 } else if (dst_first_rc == rc_float) {
2363 // gpr -> xmm
2364 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2365 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2366 // 64-bit
2367 if (masm) {
2368 __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
2369 #ifndef PRODUCT
2370 } else {
2371 st->print("movdq %s, %s\t# spill",
2372 Matcher::regName[dst_first],
2373 Matcher::regName[src_first]);
2374 #endif
2375 }
2376 } else {
2377 // 32-bit
2378 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2379 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2380 if (masm) {
2381 __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
2382 #ifndef PRODUCT
2383 } else {
2384 st->print("movdl %s, %s\t# spill",
2385 Matcher::regName[dst_first],
2386 Matcher::regName[src_first]);
2387 #endif
2388 }
2389 }
2390 return 0;
2391 } else if (dst_first_rc == rc_kreg) {
2392 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2393 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2394 // 64-bit
2395 if (masm) {
2396 __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
2397 #ifndef PRODUCT
2398 } else {
2399 st->print("kmovq %s, %s\t# spill",
2400 Matcher::regName[dst_first],
2401 Matcher::regName[src_first]);
2402 #endif
2403 }
2404 }
2405 Unimplemented();
2406 return 0;
2407 }
2408 } else if (src_first_rc == rc_float) {
2409 // xmm ->
2410 if (dst_first_rc == rc_stack) {
2411 // xmm -> mem
2412 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2413 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2414 // 64-bit
2415 int offset = ra_->reg2offset(dst_first);
2416 if (masm) {
2417 __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
2418 #ifndef PRODUCT
2419 } else {
2420 st->print("movsd [rsp + #%d], %s\t# spill",
2421 offset,
2422 Matcher::regName[src_first]);
2423 #endif
2424 }
2425 } else {
2426 // 32-bit
2427 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2428 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2429 int offset = ra_->reg2offset(dst_first);
2430 if (masm) {
2431 __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
2432 #ifndef PRODUCT
2433 } else {
2434 st->print("movss [rsp + #%d], %s\t# spill",
2435 offset,
2436 Matcher::regName[src_first]);
2437 #endif
2438 }
2439 }
2440 return 0;
2441 } else if (dst_first_rc == rc_int) {
2442 // xmm -> gpr
2443 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2444 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2445 // 64-bit
2446 if (masm) {
2447 __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2448 #ifndef PRODUCT
2449 } else {
2450 st->print("movdq %s, %s\t# spill",
2451 Matcher::regName[dst_first],
2452 Matcher::regName[src_first]);
2453 #endif
2454 }
2455 } else {
2456 // 32-bit
2457 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2458 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2459 if (masm) {
2460 __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2461 #ifndef PRODUCT
2462 } else {
2463 st->print("movdl %s, %s\t# spill",
2464 Matcher::regName[dst_first],
2465 Matcher::regName[src_first]);
2466 #endif
2467 }
2468 }
2469 return 0;
2470 } else if (dst_first_rc == rc_float) {
2471 // xmm -> xmm
2472 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2473 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2474 // 64-bit
2475 if (masm) {
2476 __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2477 #ifndef PRODUCT
2478 } else {
2479 st->print("%s %s, %s\t# spill",
2480 UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
2481 Matcher::regName[dst_first],
2482 Matcher::regName[src_first]);
2483 #endif
2484 }
2485 } else {
2486 // 32-bit
2487 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
2488 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
2489 if (masm) {
2490 __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
2491 #ifndef PRODUCT
2492 } else {
2493 st->print("%s %s, %s\t# spill",
2494 UseXmmRegToRegMoveAll ? "movaps" : "movss ",
2495 Matcher::regName[dst_first],
2496 Matcher::regName[src_first]);
2497 #endif
2498 }
2499 }
2500 return 0;
2501 } else if (dst_first_rc == rc_kreg) {
2502 assert(false, "Illegal spilling");
2503 return 0;
2504 }
2505 } else if (src_first_rc == rc_kreg) {
2506 if (dst_first_rc == rc_stack) {
2507 // mem -> kreg
2508 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2509 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2510 // 64-bit
2511 int offset = ra_->reg2offset(dst_first);
2512 if (masm) {
2513 __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
2514 #ifndef PRODUCT
2515 } else {
2516 st->print("kmovq [rsp + #%d] , %s\t# spill",
2517 offset,
2518 Matcher::regName[src_first]);
2519 #endif
2520 }
2521 }
2522 return 0;
2523 } else if (dst_first_rc == rc_int) {
2524 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2525 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2526 // 64-bit
2527 if (masm) {
2528 __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
2529 #ifndef PRODUCT
2530 } else {
2531 st->print("kmovq %s, %s\t# spill",
2532 Matcher::regName[dst_first],
2533 Matcher::regName[src_first]);
2534 #endif
2535 }
2536 }
2537 Unimplemented();
2538 return 0;
2539 } else if (dst_first_rc == rc_kreg) {
2540 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
2541 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
2542 // 64-bit
2543 if (masm) {
2544 __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
2545 #ifndef PRODUCT
2546 } else {
2547 st->print("kmovq %s, %s\t# spill",
2548 Matcher::regName[dst_first],
2549 Matcher::regName[src_first]);
2550 #endif
2551 }
2552 }
2553 return 0;
2554 } else if (dst_first_rc == rc_float) {
2555 assert(false, "Illegal spill");
2556 return 0;
2557 }
2558 }
2559
2560 assert(0," foo ");
2561 Unimplemented();
2562 return 0;
2563 }
2564
2565 #ifndef PRODUCT
2566 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
2567 implementation(nullptr, ra_, false, st);
2568 }
2569 #endif
2570
2571 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2572 implementation(masm, ra_, false, nullptr);
2573 }
2574
2575 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2576 return MachNode::size(ra_);
2577 }
2578
2579 //=============================================================================
2580 #ifndef PRODUCT
2581 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2582 {
2583 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2584 int reg = ra_->get_reg_first(this);
2585 st->print("leaq %s, [rsp + #%d]\t# box lock",
2586 Matcher::regName[reg], offset);
2587 }
2588 #endif
2589
2590 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2591 {
2592 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2593 int reg = ra_->get_encode(this);
2594
2595 __ lea(as_Register(reg), Address(rsp, offset));
2596 }
2597
2598 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
2599 {
2600 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2601 if (ra_->get_encode(this) > 15) {
2602 return (offset < 0x80) ? 6 : 9; // REX2
2603 } else {
2604 return (offset < 0x80) ? 5 : 8; // REX
2605 }
2606 }
2607
2608 //=============================================================================
2609 #ifndef PRODUCT
2610 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2611 {
2612 st->print_cr("movl rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2613 st->print_cr("\tcmpl rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
2614 st->print_cr("\tjne SharedRuntime::_ic_miss_stub");
2615 }
2616 #endif
2617
2618 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2619 {
2620 __ ic_check(InteriorEntryAlignment);
2621 }
2622
2623 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2624 {
2625 return MachNode::size(ra_); // too many variables; just compute it
2626 // the hard way
2627 }
2628
2629
2630 //=============================================================================
2631
2632 bool Matcher::supports_vector_calling_convention(void) {
2633 return EnableVectorSupport;
2634 }
2635
2636 static bool is_ndd_demotable_opr1(const MachNode* mdef) {
2637 return ((mdef->flags() & Node::PD::Flag_ndd_demotable_opr1) != 0);
2638 }
2639
2640 static bool is_ndd_demotable_opr2(const MachNode* mdef) {
2641 return ((mdef->flags() & Node::PD::Flag_ndd_demotable_opr2) != 0);
2642 }
2643
2644 #ifdef ASSERT
2645 static bool is_ndd_demotable(const MachNode* mdef) {
2646 return (is_ndd_demotable_opr1(mdef) || is_ndd_demotable_opr2(mdef));
2647 }
2648 #endif
2649
2650 bool Matcher::is_register_biasing_candidate(const MachNode* mdef,
2651 int oper_index) {
2652 if (mdef == nullptr) {
2653 return false;
2654 }
2655
2656 if (mdef->num_opnds() <= oper_index || mdef->operand_index(oper_index) < 0 ||
2657 mdef->in(mdef->operand_index(oper_index)) == nullptr) {
2658 assert(oper_index != 1 || !is_ndd_demotable_opr1(mdef), "%s", mdef->Name());
2659 assert(oper_index != 2 || !is_ndd_demotable_opr2(mdef), "%s", mdef->Name());
2660 return false;
2661 }
2662
2663 // Complex memory operand covers multiple incoming edges needed for
2664 // address computation. Biasing def towards any address component will not
2665 // result in NDD demotion by assembler.
2666 if (mdef->operand_num_edges(oper_index) != 1) {
2667 return false;
2668 }
2669
2670 // Demotion candidate must be register mask compatible with definition.
2671 const RegMask& oper_mask = mdef->in_RegMask(mdef->operand_index(oper_index));
2672 if (!oper_mask.overlap(mdef->out_RegMask())) {
2673 assert(!is_ndd_demotable(mdef), "%s", mdef->Name());
2674 return false;
2675 }
2676
2677 switch (oper_index) {
2678 // First operand of MachNode corresponding to Intel APX NDD selection
2679 // pattern can share its assigned register with definition operand if
2680 // their live ranges do not overlap. In such a scenario we can demote
2681 // it to legacy map0/map1 instruction by replacing its 4-byte extended
2682 // EVEX prefix with shorter REX/REX2 encoding. Demotion candidates
2683 // are decorated with a special flag by instruction selector.
2684 case 1:
2685 return is_ndd_demotable_opr1(mdef);
2686
2687 // Definition operand of commutative operation can be biased towards second
2688 // operand.
2689 case 2:
2690 return is_ndd_demotable_opr2(mdef);
2691
2692 // Current scheme only selects up to two biasing candidates
2693 default:
2694 assert(false, "unhandled operand index: %s", mdef->Name());
2695 break;
2696 }
2697
2698 return false;
2699 }
2700
2701 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2702 assert(EnableVectorSupport, "sanity");
2703 int lo = XMM0_num;
2704 int hi = XMM0b_num;
2705 if (ideal_reg == Op_VecX) hi = XMM0d_num;
2706 else if (ideal_reg == Op_VecY) hi = XMM0h_num;
2707 else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
2708 return OptoRegPair(hi, lo);
2709 }
2710
2711 // Is this branch offset short enough that a short branch can be used?
2712 //
2713 // NOTE: If the platform does not provide any short branch variants, then
2714 // this method should return false for offset 0.
2715 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2716 // The passed offset is relative to address of the branch.
2717 // On 86 a branch displacement is calculated relative to address
2718 // of a next instruction.
2719 offset -= br_size;
2720
2721 // the short version of jmpConUCF2 contains multiple branches,
2722 // making the reach slightly less
2723 if (rule == jmpConUCF2_rule)
2724 return (-126 <= offset && offset <= 125);
2725 return (-128 <= offset && offset <= 127);
2726 }
2727
2728 #ifdef ASSERT
2729 // Return whether or not this register is ever used as an argument.
2730 bool Matcher::can_be_java_arg(int reg)
2731 {
2732 return
2733 reg == RDI_num || reg == RDI_H_num ||
2734 reg == RSI_num || reg == RSI_H_num ||
2735 reg == RDX_num || reg == RDX_H_num ||
2736 reg == RCX_num || reg == RCX_H_num ||
2737 reg == R8_num || reg == R8_H_num ||
2738 reg == R9_num || reg == R9_H_num ||
2739 reg == R12_num || reg == R12_H_num ||
2740 reg == XMM0_num || reg == XMM0b_num ||
2741 reg == XMM1_num || reg == XMM1b_num ||
2742 reg == XMM2_num || reg == XMM2b_num ||
2743 reg == XMM3_num || reg == XMM3b_num ||
2744 reg == XMM4_num || reg == XMM4b_num ||
2745 reg == XMM5_num || reg == XMM5b_num ||
2746 reg == XMM6_num || reg == XMM6b_num ||
2747 reg == XMM7_num || reg == XMM7b_num;
2748 }
2749 #endif
2750
2751 uint Matcher::int_pressure_limit()
2752 {
2753 return (INTPRESSURE == -1) ? _INT_REG_mask.size() : INTPRESSURE;
2754 }
2755
2756 uint Matcher::float_pressure_limit()
2757 {
2758 // After experiment around with different values, the following default threshold
2759 // works best for LCM's register pressure scheduling on x64.
2760 uint dec_count = VM_Version::supports_evex() ? 4 : 2;
2761 uint default_float_pressure_threshold = _FLOAT_REG_mask.size() - dec_count;
2762 return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
2763 }
2764
2765 // Register for the first projection of an int pair
2766 const RegMask& Matcher::firstI_proj_mask() {
2767 return INT_RAX_REG_mask();
2768 }
2769
2770 // Register for the second projection of an int pair
2771 const RegMask& Matcher::secondI_proj_mask() {
2772 return INT_RDX_REG_mask();
2773 }
2774
2775 // Register for the first projection of a long pair
2776 const RegMask& Matcher::firstL_proj_mask() {
2777 return LONG_RAX_REG_mask();
2778 }
2779
2780 // Register for the second projection of a long pair
2781 const RegMask& Matcher::secondL_proj_mask() {
2782 return LONG_RDX_REG_mask();
2783 }
2784
2785 %}
2786
2787 source_hpp %{
2788 // Header information of the source block.
2789 // Method declarations/definitions which are used outside
2790 // the ad-scope can conveniently be defined here.
2791 //
2792 // To keep related declarations/definitions/uses close together,
2793 // we switch between source %{ }% and source_hpp %{ }% freely as needed.
2794
2795 #include "runtime/vm_version.hpp"
2796
2797 class NativeJump;
2798
2799 class CallStubImpl {
2800
2801 //--------------------------------------------------------------
2802 //---< Used for optimization in Compile::shorten_branches >---
2803 //--------------------------------------------------------------
2804
2805 public:
2806 // Size of call trampoline stub.
2807 static uint size_call_trampoline() {
2808 return 0; // no call trampolines on this platform
2809 }
2810
2811 // number of relocations needed by a call trampoline stub
2812 static uint reloc_call_trampoline() {
2813 return 0; // no call trampolines on this platform
2814 }
2815 };
2816
2817 class HandlerImpl {
2818
2819 public:
2820
2821 static int emit_deopt_handler(C2_MacroAssembler* masm);
2822
2823 static uint size_deopt_handler() {
2824 // one call and one jmp.
2825 return 7;
2826 }
2827 };
2828
2829 inline Assembler::AvxVectorLen vector_length_encoding(int bytes) {
2830 switch(bytes) {
2831 case 4: // fall-through
2832 case 8: // fall-through
2833 case 16: return Assembler::AVX_128bit;
2834 case 32: return Assembler::AVX_256bit;
2835 case 64: return Assembler::AVX_512bit;
2836
2837 default: {
2838 ShouldNotReachHere();
2839 return Assembler::AVX_NoVec;
2840 }
2841 }
2842 }
2843
2844 static inline Assembler::AvxVectorLen vector_length_encoding(const Node* n) {
2845 return vector_length_encoding(Matcher::vector_length_in_bytes(n));
2846 }
2847
2848 static inline Assembler::AvxVectorLen vector_length_encoding(const MachNode* use, MachOper* opnd) {
2849 uint def_idx = use->operand_index(opnd);
2850 Node* def = use->in(def_idx);
2851 return vector_length_encoding(def);
2852 }
2853
2854 static inline bool is_vector_popcount_predicate(BasicType bt) {
2855 return (is_subword_type(bt) && VM_Version::supports_avx512_bitalg()) ||
2856 (is_non_subword_integral_type(bt) && VM_Version::supports_avx512_vpopcntdq());
2857 }
2858
2859 static inline bool is_clz_non_subword_predicate_evex(BasicType bt, int vlen_bytes) {
2860 return is_non_subword_integral_type(bt) && VM_Version::supports_avx512cd() &&
2861 (VM_Version::supports_avx512vl() || vlen_bytes == 64);
2862 }
2863
2864 class Node::PD {
2865 public:
2866 enum NodeFlags : uint64_t {
2867 Flag_intel_jcc_erratum = Node::_last_flag << 1,
2868 Flag_sets_carry_flag = Node::_last_flag << 2,
2869 Flag_sets_parity_flag = Node::_last_flag << 3,
2870 Flag_sets_zero_flag = Node::_last_flag << 4,
2871 Flag_sets_overflow_flag = Node::_last_flag << 5,
2872 Flag_sets_sign_flag = Node::_last_flag << 6,
2873 Flag_clears_carry_flag = Node::_last_flag << 7,
2874 Flag_clears_parity_flag = Node::_last_flag << 8,
2875 Flag_clears_zero_flag = Node::_last_flag << 9,
2876 Flag_clears_overflow_flag = Node::_last_flag << 10,
2877 Flag_clears_sign_flag = Node::_last_flag << 11,
2878 Flag_ndd_demotable_opr1 = Node::_last_flag << 12,
2879 Flag_ndd_demotable_opr2 = Node::_last_flag << 13,
2880 _last_flag = Flag_ndd_demotable_opr2
2881 };
2882 };
2883
2884 %} // end source_hpp
2885
2886 source %{
2887
2888 #include "opto/addnode.hpp"
2889 #include "c2_intelJccErratum_x86.hpp"
2890
2891 void PhaseOutput::pd_perform_mach_node_analysis() {
2892 if (VM_Version::has_intel_jcc_erratum()) {
2893 int extra_padding = IntelJccErratum::tag_affected_machnodes(C, C->cfg(), C->regalloc());
2894 _buf_sizes._code += extra_padding;
2895 }
2896 }
2897
2898 int MachNode::pd_alignment_required() const {
2899 if (VM_Version::has_intel_jcc_erratum() && IntelJccErratum::is_jcc_erratum_branch(this)) {
2900 // Conservatively add worst case padding. We assume that relocInfo::addr_unit() is 1 on x86.
2901 return IntelJccErratum::largest_jcc_size() + 1;
2902 } else {
2903 return 1;
2904 }
2905 }
2906
2907 int MachNode::compute_padding(int current_offset) const {
2908 if (flags() & Node::PD::Flag_intel_jcc_erratum) {
2909 Compile* C = Compile::current();
2910 PhaseOutput* output = C->output();
2911 Block* block = output->block();
2912 int index = output->index();
2913 return IntelJccErratum::compute_padding(current_offset, this, block, index, C->regalloc());
2914 } else {
2915 return 0;
2916 }
2917 }
2918
2919 // Emit deopt handler code.
2920 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm) {
2921
2922 // Note that the code buffer's insts_mark is always relative to insts.
2923 // That's why we must use the macroassembler to generate a handler.
2924 address base = __ start_a_stub(size_deopt_handler());
2925 if (base == nullptr) {
2926 ciEnv::current()->record_failure("CodeCache is full");
2927 return 0; // CodeBuffer::expand failed
2928 }
2929 int offset = __ offset();
2930
2931 Label start;
2932 __ bind(start);
2933
2934 __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2935
2936 int entry_offset = __ offset();
2937
2938 __ jmp(start);
2939
2940 assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow %d", (__ offset() - offset));
2941 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2942 "out of bounds read in post-call NOP check");
2943 __ end_a_stub();
2944 return entry_offset;
2945 }
2946
2947 static Assembler::Width widthForType(BasicType bt) {
2948 if (bt == T_BYTE) {
2949 return Assembler::B;
2950 } else if (bt == T_SHORT) {
2951 return Assembler::W;
2952 } else if (bt == T_INT) {
2953 return Assembler::D;
2954 } else {
2955 assert(bt == T_LONG, "not a long: %s", type2name(bt));
2956 return Assembler::Q;
2957 }
2958 }
2959
2960 //=============================================================================
2961
2962 // Float masks come from different places depending on platform.
2963 static address float_signmask() { return StubRoutines::x86::float_sign_mask(); }
2964 static address float_signflip() { return StubRoutines::x86::float_sign_flip(); }
2965 static address double_signmask() { return StubRoutines::x86::double_sign_mask(); }
2966 static address double_signflip() { return StubRoutines::x86::double_sign_flip(); }
2967 static address vector_short_to_byte_mask() { return StubRoutines::x86::vector_short_to_byte_mask(); }
2968 static address vector_int_to_byte_mask() { return StubRoutines::x86::vector_int_to_byte_mask(); }
2969 static address vector_byte_perm_mask() { return StubRoutines::x86::vector_byte_perm_mask(); }
2970 static address vector_long_sign_mask() { return StubRoutines::x86::vector_long_sign_mask(); }
2971 static address vector_all_bits_set() { return StubRoutines::x86::vector_all_bits_set(); }
2972 static address vector_int_mask_cmp_bits() { return StubRoutines::x86::vector_int_mask_cmp_bits(); }
2973 static address vector_int_to_short_mask() { return StubRoutines::x86::vector_int_to_short_mask(); }
2974 static address vector_byte_shufflemask() { return StubRoutines::x86::vector_byte_shuffle_mask(); }
2975 static address vector_short_shufflemask() { return StubRoutines::x86::vector_short_shuffle_mask(); }
2976 static address vector_int_shufflemask() { return StubRoutines::x86::vector_int_shuffle_mask(); }
2977 static address vector_long_shufflemask() { return StubRoutines::x86::vector_long_shuffle_mask(); }
2978 static address vector_32_bit_mask() { return StubRoutines::x86::vector_32_bit_mask(); }
2979 static address vector_64_bit_mask() { return StubRoutines::x86::vector_64_bit_mask(); }
2980 static address vector_float_signflip() { return StubRoutines::x86::vector_float_sign_flip();}
2981 static address vector_double_signflip() { return StubRoutines::x86::vector_double_sign_flip();}
2982
2983 //=============================================================================
2984 bool Matcher::match_rule_supported(int opcode) {
2985 if (!has_match_rule(opcode)) {
2986 return false; // no match rule present
2987 }
2988 switch (opcode) {
2989 case Op_AbsVL:
2990 case Op_StoreVectorScatter:
2991 if (UseAVX < 3) {
2992 return false;
2993 }
2994 break;
2995 case Op_PopCountI:
2996 case Op_PopCountL:
2997 if (!UsePopCountInstruction) {
2998 return false;
2999 }
3000 break;
3001 case Op_PopCountVI:
3002 if (UseAVX < 2) {
3003 return false;
3004 }
3005 break;
3006 case Op_CompressV:
3007 case Op_ExpandV:
3008 case Op_PopCountVL:
3009 if (UseAVX < 2) {
3010 return false;
3011 }
3012 break;
3013 case Op_MulVI:
3014 if ((UseSSE < 4) && (UseAVX < 1)) { // only with SSE4_1 or AVX
3015 return false;
3016 }
3017 break;
3018 case Op_MulVL:
3019 if (UseSSE < 4) { // only with SSE4_1 or AVX
3020 return false;
3021 }
3022 break;
3023 case Op_MulReductionVL:
3024 if (VM_Version::supports_avx512dq() == false) {
3025 return false;
3026 }
3027 break;
3028 case Op_AbsVB:
3029 case Op_AbsVS:
3030 case Op_AbsVI:
3031 case Op_AddReductionVI:
3032 case Op_AndReductionV:
3033 case Op_OrReductionV:
3034 case Op_XorReductionV:
3035 if (UseSSE < 3) { // requires at least SSSE3
3036 return false;
3037 }
3038 break;
3039 case Op_MaxHF:
3040 case Op_MinHF:
3041 if (!VM_Version::supports_avx512vlbw()) {
3042 return false;
3043 } // fallthrough
3044 case Op_AddHF:
3045 case Op_DivHF:
3046 case Op_FmaHF:
3047 case Op_MulHF:
3048 case Op_ReinterpretS2HF:
3049 case Op_ReinterpretHF2S:
3050 case Op_SubHF:
3051 case Op_SqrtHF:
3052 if (!VM_Version::supports_avx512_fp16()) {
3053 return false;
3054 }
3055 break;
3056 case Op_VectorLoadShuffle:
3057 case Op_VectorRearrange:
3058 case Op_MulReductionVI:
3059 if (UseSSE < 4) { // requires at least SSE4
3060 return false;
3061 }
3062 break;
3063 case Op_IsInfiniteF:
3064 case Op_IsInfiniteD:
3065 if (!VM_Version::supports_avx512dq()) {
3066 return false;
3067 }
3068 break;
3069 case Op_SqrtVD:
3070 case Op_SqrtVF:
3071 case Op_VectorMaskCmp:
3072 case Op_VectorCastB2X:
3073 case Op_VectorCastS2X:
3074 case Op_VectorCastI2X:
3075 case Op_VectorCastL2X:
3076 case Op_VectorCastF2X:
3077 case Op_VectorCastD2X:
3078 case Op_VectorUCastB2X:
3079 case Op_VectorUCastS2X:
3080 case Op_VectorUCastI2X:
3081 case Op_VectorMaskCast:
3082 if (UseAVX < 1) { // enabled for AVX only
3083 return false;
3084 }
3085 break;
3086 case Op_PopulateIndex:
3087 if (UseAVX < 2) {
3088 return false;
3089 }
3090 break;
3091 case Op_RoundVF:
3092 if (UseAVX < 2) { // enabled for AVX2 only
3093 return false;
3094 }
3095 break;
3096 case Op_RoundVD:
3097 if (UseAVX < 3) {
3098 return false; // enabled for AVX3 only
3099 }
3100 break;
3101 case Op_CompareAndSwapL:
3102 case Op_CompareAndSwapP:
3103 break;
3104 case Op_StrIndexOf:
3105 if (!UseSSE42Intrinsics) {
3106 return false;
3107 }
3108 break;
3109 case Op_StrIndexOfChar:
3110 if (!UseSSE42Intrinsics) {
3111 return false;
3112 }
3113 break;
3114 case Op_OnSpinWait:
3115 if (VM_Version::supports_on_spin_wait() == false) {
3116 return false;
3117 }
3118 break;
3119 case Op_MulVB:
3120 case Op_LShiftVB:
3121 case Op_RShiftVB:
3122 case Op_URShiftVB:
3123 case Op_VectorInsert:
3124 case Op_VectorLoadMask:
3125 case Op_VectorStoreMask:
3126 case Op_VectorBlend:
3127 if (UseSSE < 4) {
3128 return false;
3129 }
3130 break;
3131 case Op_MaxD:
3132 case Op_MaxF:
3133 case Op_MinD:
3134 case Op_MinF:
3135 if (UseAVX < 1) { // enabled for AVX only
3136 return false;
3137 }
3138 break;
3139 case Op_CacheWB:
3140 case Op_CacheWBPreSync:
3141 case Op_CacheWBPostSync:
3142 if (!VM_Version::supports_data_cache_line_flush()) {
3143 return false;
3144 }
3145 break;
3146 case Op_ExtractB:
3147 case Op_ExtractL:
3148 case Op_ExtractI:
3149 case Op_RoundDoubleMode:
3150 if (UseSSE < 4) {
3151 return false;
3152 }
3153 break;
3154 case Op_RoundDoubleModeV:
3155 if (VM_Version::supports_avx() == false) {
3156 return false; // 128bit vroundpd is not available
3157 }
3158 break;
3159 case Op_LoadVectorGather:
3160 case Op_LoadVectorGatherMasked:
3161 if (UseAVX < 2) {
3162 return false;
3163 }
3164 break;
3165 case Op_FmaF:
3166 case Op_FmaD:
3167 case Op_FmaVD:
3168 case Op_FmaVF:
3169 if (!UseFMA) {
3170 return false;
3171 }
3172 break;
3173 case Op_MacroLogicV:
3174 if (UseAVX < 3 || !UseVectorMacroLogic) {
3175 return false;
3176 }
3177 break;
3178
3179 case Op_VectorCmpMasked:
3180 if (UseAVX < 3 || !UseCountTrailingZerosInstruction) {
3181 return false;
3182 }
3183 break;
3184 case Op_VectorMaskGen:
3185 if (UseAVX < 3 || !VM_Version::supports_bmi2()) {
3186 return false;
3187 }
3188 break;
3189 case Op_VectorMaskFirstTrue:
3190 case Op_VectorMaskLastTrue:
3191 case Op_VectorMaskTrueCount:
3192 case Op_VectorMaskToLong:
3193 if (UseAVX < 1) {
3194 return false;
3195 }
3196 break;
3197 case Op_RoundF:
3198 case Op_RoundD:
3199 break;
3200 case Op_CopySignD:
3201 case Op_CopySignF:
3202 if (UseAVX < 3) {
3203 return false;
3204 }
3205 if (!VM_Version::supports_avx512vl()) {
3206 return false;
3207 }
3208 break;
3209 case Op_CompressBits:
3210 case Op_ExpandBits:
3211 if (!VM_Version::supports_bmi2()) {
3212 return false;
3213 }
3214 break;
3215 case Op_CompressM:
3216 if (!VM_Version::supports_avx512vl() || !VM_Version::supports_bmi2()) {
3217 return false;
3218 }
3219 break;
3220 case Op_ConvF2HF:
3221 case Op_ConvHF2F:
3222 if (!VM_Version::supports_float16()) {
3223 return false;
3224 }
3225 break;
3226 case Op_VectorCastF2HF:
3227 case Op_VectorCastHF2F:
3228 if (!VM_Version::supports_f16c() && !VM_Version::supports_evex()) {
3229 return false;
3230 }
3231 break;
3232 }
3233 return true; // Match rules are supported by default.
3234 }
3235
3236 //------------------------------------------------------------------------
3237
3238 static inline bool is_pop_count_instr_target(BasicType bt) {
3239 return (is_subword_type(bt) && VM_Version::supports_avx512_bitalg()) ||
3240 (is_non_subword_integral_type(bt) && VM_Version::supports_avx512_vpopcntdq());
3241 }
3242
3243 bool Matcher::match_rule_supported_auto_vectorization(int opcode, int vlen, BasicType bt) {
3244 return match_rule_supported_vector(opcode, vlen, bt);
3245 }
3246
3247 // Identify extra cases that we might want to provide match rules for vector nodes and
3248 // other intrinsics guarded with vector length (vlen) and element type (bt).
3249 bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
3250 if (!match_rule_supported(opcode)) {
3251 return false;
3252 }
3253 // Matcher::vector_size_supported() restricts vector sizes in the following way (see Matcher::vector_width_in_bytes):
3254 // * SSE2 supports 128bit vectors for all types;
3255 // * AVX1 supports 256bit vectors only for FLOAT and DOUBLE types;
3256 // * AVX2 supports 256bit vectors for all types;
3257 // * AVX512F supports 512bit vectors only for INT, FLOAT, and DOUBLE types;
3258 // * AVX512BW supports 512bit vectors for BYTE, SHORT, and CHAR types.
3259 // There's also a limit on minimum vector size supported: 2 elements (or 4 bytes for BYTE).
3260 // And MaxVectorSize is taken into account as well.
3261 if (!vector_size_supported(bt, vlen)) {
3262 return false;
3263 }
3264 // Special cases which require vector length follow:
3265 // * implementation limitations
3266 // * some 512bit vector operations on FLOAT and DOUBLE types require AVX512DQ
3267 // * 128bit vroundpd instruction is present only in AVX1
3268 int size_in_bits = vlen * type2aelembytes(bt) * BitsPerByte;
3269 switch (opcode) {
3270 case Op_MaxVHF:
3271 case Op_MinVHF:
3272 if (!VM_Version::supports_avx512bw()) {
3273 return false;
3274 }
3275 case Op_AddVHF:
3276 case Op_DivVHF:
3277 case Op_FmaVHF:
3278 case Op_MulVHF:
3279 case Op_SubVHF:
3280 case Op_SqrtVHF:
3281 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3282 return false;
3283 }
3284 if (!VM_Version::supports_avx512_fp16()) {
3285 return false;
3286 }
3287 break;
3288 case Op_AbsVF:
3289 case Op_NegVF:
3290 if ((vlen == 16) && (VM_Version::supports_avx512dq() == false)) {
3291 return false; // 512bit vandps and vxorps are not available
3292 }
3293 break;
3294 case Op_AbsVD:
3295 case Op_NegVD:
3296 if ((vlen == 8) && (VM_Version::supports_avx512dq() == false)) {
3297 return false; // 512bit vpmullq, vandpd and vxorpd are not available
3298 }
3299 break;
3300 case Op_RotateRightV:
3301 case Op_RotateLeftV:
3302 if (bt != T_INT && bt != T_LONG) {
3303 return false;
3304 } // fallthrough
3305 case Op_MacroLogicV:
3306 if (!VM_Version::supports_evex() ||
3307 ((size_in_bits != 512) && !VM_Version::supports_avx512vl())) {
3308 return false;
3309 }
3310 break;
3311 case Op_ClearArray:
3312 case Op_VectorMaskGen:
3313 case Op_VectorCmpMasked:
3314 if (!VM_Version::supports_avx512bw()) {
3315 return false;
3316 }
3317 if ((size_in_bits != 512) && !VM_Version::supports_avx512vl()) {
3318 return false;
3319 }
3320 break;
3321 case Op_LoadVectorMasked:
3322 case Op_StoreVectorMasked:
3323 if (!VM_Version::supports_avx512bw() && (is_subword_type(bt) || UseAVX < 1)) {
3324 return false;
3325 }
3326 break;
3327 case Op_UMinV:
3328 case Op_UMaxV:
3329 if (UseAVX == 0) {
3330 return false;
3331 }
3332 break;
3333 case Op_UMinReductionV:
3334 case Op_UMaxReductionV:
3335 if (UseAVX == 0) {
3336 return false;
3337 }
3338 if (bt == T_LONG && !VM_Version::supports_avx512vl()) {
3339 return false;
3340 }
3341 if (UseAVX > 2 && size_in_bits == 512 && !VM_Version::supports_avx512vl()) {
3342 return false;
3343 }
3344 break;
3345 case Op_MaxV:
3346 case Op_MinV:
3347 if (UseSSE < 4 && is_integral_type(bt)) {
3348 return false;
3349 }
3350 if ((bt == T_FLOAT || bt == T_DOUBLE)) {
3351 // Float/Double intrinsics are enabled for AVX family currently.
3352 if (UseAVX == 0) {
3353 return false;
3354 }
3355 if (UseAVX > 2 && (!VM_Version::supports_avx512dq() && size_in_bits == 512)) { // 512 bit Float/Double intrinsics need AVX512DQ
3356 return false;
3357 }
3358 }
3359 break;
3360 case Op_CallLeafVector:
3361 if (size_in_bits == 512 && !VM_Version::supports_avx512vlbwdq()) {
3362 return false;
3363 }
3364 break;
3365 case Op_AddReductionVI:
3366 if (bt == T_INT && (UseSSE < 3 || !VM_Version::supports_ssse3())) {
3367 return false;
3368 }
3369 // fallthrough
3370 case Op_AndReductionV:
3371 case Op_OrReductionV:
3372 case Op_XorReductionV:
3373 if (is_subword_type(bt) && (UseSSE < 4)) {
3374 return false;
3375 }
3376 break;
3377 case Op_MinReductionV:
3378 case Op_MaxReductionV:
3379 if ((bt == T_INT || is_subword_type(bt)) && UseSSE < 4) {
3380 return false;
3381 } else if (bt == T_LONG && (UseAVX < 3 || !VM_Version::supports_avx512vlbwdq())) {
3382 return false;
3383 }
3384 // Float/Double intrinsics enabled for AVX family.
3385 if (UseAVX == 0 && (bt == T_FLOAT || bt == T_DOUBLE)) {
3386 return false;
3387 }
3388 if (UseAVX > 2 && (!VM_Version::supports_avx512dq() && size_in_bits == 512)) {
3389 return false;
3390 }
3391 break;
3392 case Op_VectorBlend:
3393 if (UseAVX == 0 && size_in_bits < 128) {
3394 return false;
3395 }
3396 break;
3397 case Op_VectorTest:
3398 if (UseSSE < 4) {
3399 return false; // Implementation limitation
3400 } else if (size_in_bits < 32) {
3401 return false; // Implementation limitation
3402 }
3403 break;
3404 case Op_VectorLoadShuffle:
3405 case Op_VectorRearrange:
3406 if(vlen == 2) {
3407 return false; // Implementation limitation due to how shuffle is loaded
3408 } else if (size_in_bits == 256 && UseAVX < 2) {
3409 return false; // Implementation limitation
3410 }
3411 break;
3412 case Op_VectorLoadMask:
3413 case Op_VectorMaskCast:
3414 if (size_in_bits == 256 && UseAVX < 2) {
3415 return false; // Implementation limitation
3416 }
3417 // fallthrough
3418 case Op_VectorStoreMask:
3419 if (vlen == 2) {
3420 return false; // Implementation limitation
3421 }
3422 break;
3423 case Op_PopulateIndex:
3424 if (size_in_bits > 256 && !VM_Version::supports_avx512bw()) {
3425 return false;
3426 }
3427 break;
3428 case Op_VectorCastB2X:
3429 case Op_VectorCastS2X:
3430 case Op_VectorCastI2X:
3431 if (bt != T_DOUBLE && size_in_bits == 256 && UseAVX < 2) {
3432 return false;
3433 }
3434 break;
3435 case Op_VectorCastL2X:
3436 if (is_integral_type(bt) && size_in_bits == 256 && UseAVX < 2) {
3437 return false;
3438 } else if (!is_integral_type(bt) && !VM_Version::supports_avx512dq()) {
3439 return false;
3440 }
3441 break;
3442 case Op_VectorCastF2X: {
3443 // As per JLS section 5.1.3 narrowing conversion to sub-word types
3444 // happen after intermediate conversion to integer and special handling
3445 // code needs AVX2 vpcmpeqd instruction for 256 bit vectors.
3446 int src_size_in_bits = type2aelembytes(T_FLOAT) * vlen * BitsPerByte;
3447 if (is_integral_type(bt) && src_size_in_bits == 256 && UseAVX < 2) {
3448 return false;
3449 }
3450 }
3451 // fallthrough
3452 case Op_VectorCastD2X:
3453 if (bt == T_LONG && !VM_Version::supports_avx512dq()) {
3454 return false;
3455 }
3456 break;
3457 case Op_VectorCastF2HF:
3458 case Op_VectorCastHF2F:
3459 if (!VM_Version::supports_f16c() &&
3460 ((!VM_Version::supports_evex() ||
3461 ((size_in_bits != 512) && !VM_Version::supports_avx512vl())))) {
3462 return false;
3463 }
3464 break;
3465 case Op_RoundVD:
3466 if (!VM_Version::supports_avx512dq()) {
3467 return false;
3468 }
3469 break;
3470 case Op_MulReductionVI:
3471 if (bt == T_BYTE && size_in_bits == 512 && !VM_Version::supports_avx512bw()) {
3472 return false;
3473 }
3474 break;
3475 case Op_LoadVectorGatherMasked:
3476 if (!is_subword_type(bt) && size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3477 return false;
3478 }
3479 if (is_subword_type(bt) &&
3480 ((size_in_bits > 256 && !VM_Version::supports_avx512bw()) ||
3481 (size_in_bits < 64) ||
3482 (bt == T_SHORT && !VM_Version::supports_bmi2()))) {
3483 return false;
3484 }
3485 break;
3486 case Op_StoreVectorScatterMasked:
3487 case Op_StoreVectorScatter:
3488 if (is_subword_type(bt)) {
3489 return false;
3490 } else if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3491 return false;
3492 }
3493 // fallthrough
3494 case Op_LoadVectorGather:
3495 if (!is_subword_type(bt) && size_in_bits == 64) {
3496 return false;
3497 }
3498 if (is_subword_type(bt) && size_in_bits < 64) {
3499 return false;
3500 }
3501 break;
3502 case Op_SaturatingAddV:
3503 case Op_SaturatingSubV:
3504 if (UseAVX < 1) {
3505 return false; // Implementation limitation
3506 }
3507 if (is_subword_type(bt) && size_in_bits == 512 && !VM_Version::supports_avx512bw()) {
3508 return false;
3509 }
3510 break;
3511 case Op_SelectFromTwoVector:
3512 if (size_in_bits < 128) {
3513 return false;
3514 }
3515 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3516 return false;
3517 }
3518 if (bt == T_SHORT && !VM_Version::supports_avx512bw()) {
3519 return false;
3520 }
3521 if (bt == T_BYTE && !VM_Version::supports_avx512_vbmi()) {
3522 return false;
3523 }
3524 if ((bt == T_INT || bt == T_FLOAT || bt == T_DOUBLE) && !VM_Version::supports_evex()) {
3525 return false;
3526 }
3527 break;
3528 case Op_MaskAll:
3529 if (!VM_Version::supports_evex()) {
3530 return false;
3531 }
3532 if ((vlen > 16 || is_subword_type(bt)) && !VM_Version::supports_avx512bw()) {
3533 return false;
3534 }
3535 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3536 return false;
3537 }
3538 break;
3539 case Op_VectorMaskCmp:
3540 if (vlen < 2 || size_in_bits < 32) {
3541 return false;
3542 }
3543 break;
3544 case Op_CompressM:
3545 if (UseAVX < 3 || !VM_Version::supports_bmi2()) {
3546 return false;
3547 }
3548 break;
3549 case Op_CompressV:
3550 case Op_ExpandV:
3551 if (is_subword_type(bt) && !VM_Version::supports_avx512_vbmi2()) {
3552 return false;
3553 }
3554 if (size_in_bits < 128 ) {
3555 return false;
3556 }
3557 case Op_VectorLongToMask:
3558 if (UseAVX < 1) {
3559 return false;
3560 }
3561 if (UseAVX < 3 && !VM_Version::supports_bmi2()) {
3562 return false;
3563 }
3564 break;
3565 case Op_SignumVD:
3566 case Op_SignumVF:
3567 if (UseAVX < 1) {
3568 return false;
3569 }
3570 break;
3571 case Op_PopCountVI:
3572 case Op_PopCountVL: {
3573 if (!is_pop_count_instr_target(bt) &&
3574 (size_in_bits == 512) && !VM_Version::supports_avx512bw()) {
3575 return false;
3576 }
3577 }
3578 break;
3579 case Op_ReverseV:
3580 case Op_ReverseBytesV:
3581 if (UseAVX < 2) {
3582 return false;
3583 }
3584 break;
3585 case Op_CountTrailingZerosV:
3586 case Op_CountLeadingZerosV:
3587 if (UseAVX < 2) {
3588 return false;
3589 }
3590 break;
3591 }
3592 return true; // Per default match rules are supported.
3593 }
3594
3595 bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
3596 // ADLC based match_rule_supported routine checks for the existence of pattern based
3597 // on IR opcode. Most of the unary/binary/ternary masked operation share the IR nodes
3598 // of their non-masked counterpart with mask edge being the differentiator.
3599 // This routine does a strict check on the existence of masked operation patterns
3600 // by returning a default false value for all the other opcodes apart from the
3601 // ones whose masked instruction patterns are defined in this file.
3602 if (!match_rule_supported_vector(opcode, vlen, bt)) {
3603 return false;
3604 }
3605
3606 int size_in_bits = vlen * type2aelembytes(bt) * BitsPerByte;
3607 if (size_in_bits != 512 && !VM_Version::supports_avx512vl()) {
3608 return false;
3609 }
3610 switch(opcode) {
3611 // Unary masked operations
3612 case Op_AbsVB:
3613 case Op_AbsVS:
3614 if(!VM_Version::supports_avx512bw()) {
3615 return false; // Implementation limitation
3616 }
3617 case Op_AbsVI:
3618 case Op_AbsVL:
3619 return true;
3620
3621 // Ternary masked operations
3622 case Op_FmaVF:
3623 case Op_FmaVD:
3624 return true;
3625
3626 case Op_MacroLogicV:
3627 if(bt != T_INT && bt != T_LONG) {
3628 return false;
3629 }
3630 return true;
3631
3632 // Binary masked operations
3633 case Op_AddVB:
3634 case Op_AddVS:
3635 case Op_SubVB:
3636 case Op_SubVS:
3637 case Op_MulVS:
3638 case Op_LShiftVS:
3639 case Op_RShiftVS:
3640 case Op_URShiftVS:
3641 assert(size_in_bits == 512 || VM_Version::supports_avx512vl(), "");
3642 if (!VM_Version::supports_avx512bw()) {
3643 return false; // Implementation limitation
3644 }
3645 return true;
3646
3647 case Op_MulVL:
3648 assert(size_in_bits == 512 || VM_Version::supports_avx512vl(), "");
3649 if (!VM_Version::supports_avx512dq()) {
3650 return false; // Implementation limitation
3651 }
3652 return true;
3653
3654 case Op_AndV:
3655 case Op_OrV:
3656 case Op_XorV:
3657 case Op_RotateRightV:
3658 case Op_RotateLeftV:
3659 if (bt != T_INT && bt != T_LONG) {
3660 return false; // Implementation limitation
3661 }
3662 return true;
3663
3664 case Op_VectorLoadMask:
3665 assert(size_in_bits == 512 || VM_Version::supports_avx512vl(), "");
3666 if (is_subword_type(bt) && !VM_Version::supports_avx512bw()) {
3667 return false;
3668 }
3669 return true;
3670
3671 case Op_AddVI:
3672 case Op_AddVL:
3673 case Op_AddVF:
3674 case Op_AddVD:
3675 case Op_SubVI:
3676 case Op_SubVL:
3677 case Op_SubVF:
3678 case Op_SubVD:
3679 case Op_MulVI:
3680 case Op_MulVF:
3681 case Op_MulVD:
3682 case Op_DivVF:
3683 case Op_DivVD:
3684 case Op_SqrtVF:
3685 case Op_SqrtVD:
3686 case Op_LShiftVI:
3687 case Op_LShiftVL:
3688 case Op_RShiftVI:
3689 case Op_RShiftVL:
3690 case Op_URShiftVI:
3691 case Op_URShiftVL:
3692 case Op_LoadVectorMasked:
3693 case Op_StoreVectorMasked:
3694 case Op_LoadVectorGatherMasked:
3695 case Op_StoreVectorScatterMasked:
3696 return true;
3697
3698 case Op_UMinV:
3699 case Op_UMaxV:
3700 if (size_in_bits < 512 && !VM_Version::supports_avx512vl()) {
3701 return false;
3702 } // fallthrough
3703 case Op_MaxV:
3704 case Op_MinV:
3705 if (is_subword_type(bt) && !VM_Version::supports_avx512bw()) {
3706 return false; // Implementation limitation
3707 }
3708 if (is_floating_point_type(bt) && !VM_Version::supports_avx10_2()) {
3709 return false; // Implementation limitation
3710 }
3711 return true;
3712 case Op_SaturatingAddV:
3713 case Op_SaturatingSubV:
3714 if (!is_subword_type(bt)) {
3715 return false;
3716 }
3717 if (size_in_bits < 128 || !VM_Version::supports_avx512bw()) {
3718 return false; // Implementation limitation
3719 }
3720 return true;
3721
3722 case Op_VectorMaskCmp:
3723 if (is_subword_type(bt) && !VM_Version::supports_avx512bw()) {
3724 return false; // Implementation limitation
3725 }
3726 return true;
3727
3728 case Op_VectorRearrange:
3729 if (bt == T_SHORT && !VM_Version::supports_avx512bw()) {
3730 return false; // Implementation limitation
3731 }
3732 if (bt == T_BYTE && !VM_Version::supports_avx512_vbmi()) {
3733 return false; // Implementation limitation
3734 } else if ((bt == T_INT || bt == T_FLOAT) && size_in_bits < 256) {
3735 return false; // Implementation limitation
3736 }
3737 return true;
3738
3739 // Binary Logical operations
3740 case Op_AndVMask:
3741 case Op_OrVMask:
3742 case Op_XorVMask:
3743 if (vlen > 16 && !VM_Version::supports_avx512bw()) {
3744 return false; // Implementation limitation
3745 }
3746 return true;
3747
3748 case Op_PopCountVI:
3749 case Op_PopCountVL:
3750 if (!is_pop_count_instr_target(bt)) {
3751 return false;
3752 }
3753 return true;
3754
3755 case Op_MaskAll:
3756 return true;
3757
3758 case Op_CountLeadingZerosV:
3759 if (is_non_subword_integral_type(bt) && VM_Version::supports_avx512cd()) {
3760 return true;
3761 }
3762 default:
3763 return false;
3764 }
3765 }
3766
3767 bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* vt) {
3768 return false;
3769 }
3770
3771 // Return true if Vector::rearrange needs preparation of the shuffle argument
3772 bool Matcher::vector_rearrange_requires_load_shuffle(BasicType elem_bt, int vlen) {
3773 switch (elem_bt) {
3774 case T_BYTE: return false;
3775 case T_SHORT: return !VM_Version::supports_avx512bw();
3776 case T_INT: return !VM_Version::supports_avx();
3777 case T_LONG: return vlen < 8 && !VM_Version::supports_avx512vl();
3778 default:
3779 ShouldNotReachHere();
3780 return false;
3781 }
3782 }
3783
3784 bool Matcher::mask_op_prefers_predicate(int opcode, const TypeVect* vt) {
3785 // Prefer predicate if the mask type is "TypePVectMask".
3786 return vt->isa_pvectmask() != nullptr;
3787 }
3788
3789 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
3790 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
3791 bool legacy = (generic_opnd->opcode() == LEGVEC);
3792 if (!VM_Version::supports_avx512vlbwdq() && // KNL
3793 is_temp && !legacy && (ideal_reg == Op_VecZ)) {
3794 // Conservatively specialize 512bit vec TEMP operands to legVecZ (zmm0-15) on KNL.
3795 return new legVecZOper();
3796 }
3797 if (legacy) {
3798 switch (ideal_reg) {
3799 case Op_VecS: return new legVecSOper();
3800 case Op_VecD: return new legVecDOper();
3801 case Op_VecX: return new legVecXOper();
3802 case Op_VecY: return new legVecYOper();
3803 case Op_VecZ: return new legVecZOper();
3804 }
3805 } else {
3806 switch (ideal_reg) {
3807 case Op_VecS: return new vecSOper();
3808 case Op_VecD: return new vecDOper();
3809 case Op_VecX: return new vecXOper();
3810 case Op_VecY: return new vecYOper();
3811 case Op_VecZ: return new vecZOper();
3812 }
3813 }
3814 ShouldNotReachHere();
3815 return nullptr;
3816 }
3817
3818 bool Matcher::is_reg2reg_move(MachNode* m) {
3819 switch (m->rule()) {
3820 case MoveVec2Leg_rule:
3821 case MoveLeg2Vec_rule:
3822 case MoveF2VL_rule:
3823 case MoveF2LEG_rule:
3824 case MoveVL2F_rule:
3825 case MoveLEG2F_rule:
3826 case MoveD2VL_rule:
3827 case MoveD2LEG_rule:
3828 case MoveVL2D_rule:
3829 case MoveLEG2D_rule:
3830 return true;
3831 default:
3832 return false;
3833 }
3834 }
3835
3836 bool Matcher::is_generic_vector(MachOper* opnd) {
3837 switch (opnd->opcode()) {
3838 case VEC:
3839 case LEGVEC:
3840 return true;
3841 default:
3842 return false;
3843 }
3844 }
3845
3846 //------------------------------------------------------------------------
3847
3848 const RegMask* Matcher::predicate_reg_mask(void) {
3849 return &_VECTMASK_REG_mask;
3850 }
3851
3852 // Max vector size in bytes. 0 if not supported.
3853 int Matcher::vector_width_in_bytes(BasicType bt) {
3854 assert(is_java_primitive(bt), "only primitive type vectors");
3855 // SSE2 supports 128bit vectors for all types.
3856 // AVX2 supports 256bit vectors for all types.
3857 // AVX2/EVEX supports 512bit vectors for all types.
3858 int size = (UseAVX > 1) ? (1 << UseAVX) * 8 : 16;
3859 // AVX1 supports 256bit vectors only for FLOAT and DOUBLE.
3860 if (UseAVX > 0 && (bt == T_FLOAT || bt == T_DOUBLE))
3861 size = (UseAVX > 2) ? 64 : 32;
3862 if (UseAVX > 2 && (bt == T_BYTE || bt == T_SHORT || bt == T_CHAR))
3863 size = (VM_Version::supports_avx512bw()) ? 64 : 32;
3864 // Use flag to limit vector size.
3865 size = MIN2(size,(int)MaxVectorSize);
3866 // Minimum 2 values in vector (or 4 for bytes).
3867 switch (bt) {
3868 case T_DOUBLE:
3869 case T_LONG:
3870 if (size < 16) return 0;
3871 break;
3872 case T_FLOAT:
3873 case T_INT:
3874 if (size < 8) return 0;
3875 break;
3876 case T_BOOLEAN:
3877 if (size < 4) return 0;
3878 break;
3879 case T_CHAR:
3880 if (size < 4) return 0;
3881 break;
3882 case T_BYTE:
3883 if (size < 4) return 0;
3884 break;
3885 case T_SHORT:
3886 if (size < 4) return 0;
3887 break;
3888 default:
3889 ShouldNotReachHere();
3890 }
3891 return size;
3892 }
3893
3894 // Limits on vector size (number of elements) loaded into vector.
3895 int Matcher::max_vector_size(const BasicType bt) {
3896 return vector_width_in_bytes(bt)/type2aelembytes(bt);
3897 }
3898 int Matcher::min_vector_size(const BasicType bt) {
3899 int max_size = max_vector_size(bt);
3900 // Min size which can be loaded into vector is 4 bytes.
3901 int size = (type2aelembytes(bt) == 1) ? 4 : 2;
3902 // Support for calling svml double64 vectors
3903 if (bt == T_DOUBLE) {
3904 size = 1;
3905 }
3906 return MIN2(size,max_size);
3907 }
3908
3909 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
3910 // Limit the max vector size for auto vectorization to 256 bits (32 bytes)
3911 // by default on Cascade Lake
3912 if (VM_Version::is_default_intel_cascade_lake()) {
3913 return MIN2(Matcher::max_vector_size(bt), 32 / type2aelembytes(bt));
3914 }
3915 return Matcher::max_vector_size(bt);
3916 }
3917
3918 int Matcher::scalable_vector_reg_size(const BasicType bt) {
3919 return -1;
3920 }
3921
3922 // Vector ideal reg corresponding to specified size in bytes
3923 uint Matcher::vector_ideal_reg(int size) {
3924 assert(MaxVectorSize >= size, "");
3925 switch(size) {
3926 case 4: return Op_VecS;
3927 case 8: return Op_VecD;
3928 case 16: return Op_VecX;
3929 case 32: return Op_VecY;
3930 case 64: return Op_VecZ;
3931 }
3932 ShouldNotReachHere();
3933 return 0;
3934 }
3935
3936 // Check for shift by small constant as well
3937 static bool clone_shift(Node* shift, Matcher* matcher, Matcher::MStack& mstack, VectorSet& address_visited) {
3938 if (shift->Opcode() == Op_LShiftX && shift->in(2)->is_Con() &&
3939 shift->in(2)->get_int() <= 3 &&
3940 // Are there other uses besides address expressions?
3941 !matcher->is_visited(shift)) {
3942 address_visited.set(shift->_idx); // Flag as address_visited
3943 mstack.push(shift->in(2), Matcher::Visit);
3944 Node *conv = shift->in(1);
3945 // Allow Matcher to match the rule which bypass
3946 // ConvI2L operation for an array index on LP64
3947 // if the index value is positive.
3948 if (conv->Opcode() == Op_ConvI2L &&
3949 conv->as_Type()->type()->is_long()->_lo >= 0 &&
3950 // Are there other uses besides address expressions?
3951 !matcher->is_visited(conv)) {
3952 address_visited.set(conv->_idx); // Flag as address_visited
3953 mstack.push(conv->in(1), Matcher::Pre_Visit);
3954 } else {
3955 mstack.push(conv, Matcher::Pre_Visit);
3956 }
3957 return true;
3958 }
3959 return false;
3960 }
3961
3962 // This function identifies sub-graphs in which a 'load' node is
3963 // input to two different nodes, and such that it can be matched
3964 // with BMI instructions like blsi, blsr, etc.
3965 // Example : for b = -a[i] & a[i] can be matched to blsi r32, m32.
3966 // The graph is (AndL (SubL Con0 LoadL*) LoadL*), where LoadL*
3967 // refers to the same node.
3968 //
3969 // Match the generic fused operations pattern (op1 (op2 Con{ConType} mop) mop)
3970 // This is a temporary solution until we make DAGs expressible in ADL.
3971 template<typename ConType>
3972 class FusedPatternMatcher {
3973 Node* _op1_node;
3974 Node* _mop_node;
3975 int _con_op;
3976
3977 static int match_next(Node* n, int next_op, int next_op_idx) {
3978 if (n->in(1) == nullptr || n->in(2) == nullptr) {
3979 return -1;
3980 }
3981
3982 if (next_op_idx == -1) { // n is commutative, try rotations
3983 if (n->in(1)->Opcode() == next_op) {
3984 return 1;
3985 } else if (n->in(2)->Opcode() == next_op) {
3986 return 2;
3987 }
3988 } else {
3989 assert(next_op_idx > 0 && next_op_idx <= 2, "Bad argument index");
3990 if (n->in(next_op_idx)->Opcode() == next_op) {
3991 return next_op_idx;
3992 }
3993 }
3994 return -1;
3995 }
3996
3997 public:
3998 FusedPatternMatcher(Node* op1_node, Node* mop_node, int con_op) :
3999 _op1_node(op1_node), _mop_node(mop_node), _con_op(con_op) { }
4000
4001 bool match(int op1, int op1_op2_idx, // op1 and the index of the op1->op2 edge, -1 if op1 is commutative
4002 int op2, int op2_con_idx, // op2 and the index of the op2->con edge, -1 if op2 is commutative
4003 typename ConType::NativeType con_value) {
4004 if (_op1_node->Opcode() != op1) {
4005 return false;
4006 }
4007 if (_mop_node->outcnt() > 2) {
4008 return false;
4009 }
4010 op1_op2_idx = match_next(_op1_node, op2, op1_op2_idx);
4011 if (op1_op2_idx == -1) {
4012 return false;
4013 }
4014 // Memory operation must be the other edge
4015 int op1_mop_idx = (op1_op2_idx & 1) + 1;
4016
4017 // Check that the mop node is really what we want
4018 if (_op1_node->in(op1_mop_idx) == _mop_node) {
4019 Node* op2_node = _op1_node->in(op1_op2_idx);
4020 if (op2_node->outcnt() > 1) {
4021 return false;
4022 }
4023 assert(op2_node->Opcode() == op2, "Should be");
4024 op2_con_idx = match_next(op2_node, _con_op, op2_con_idx);
4025 if (op2_con_idx == -1) {
4026 return false;
4027 }
4028 // Memory operation must be the other edge
4029 int op2_mop_idx = (op2_con_idx & 1) + 1;
4030 // Check that the memory operation is the same node
4031 if (op2_node->in(op2_mop_idx) == _mop_node) {
4032 // Now check the constant
4033 const Type* con_type = op2_node->in(op2_con_idx)->bottom_type();
4034 if (con_type != Type::TOP && ConType::as_self(con_type)->get_con() == con_value) {
4035 return true;
4036 }
4037 }
4038 }
4039 return false;
4040 }
4041 };
4042
4043 static bool is_bmi_pattern(Node* n, Node* m) {
4044 assert(VM_Version::supports_bmi1() && VM_Version::supports_avx(), "sanity");
4045 if (n != nullptr && m != nullptr) {
4046 if (m->Opcode() == Op_LoadI) {
4047 FusedPatternMatcher<TypeInt> bmii(n, m, Op_ConI);
4048 return bmii.match(Op_AndI, -1, Op_SubI, 1, 0) ||
4049 bmii.match(Op_AndI, -1, Op_AddI, -1, -1) ||
4050 bmii.match(Op_XorI, -1, Op_AddI, -1, -1);
4051 } else if (m->Opcode() == Op_LoadL) {
4052 FusedPatternMatcher<TypeLong> bmil(n, m, Op_ConL);
4053 return bmil.match(Op_AndL, -1, Op_SubL, 1, 0) ||
4054 bmil.match(Op_AndL, -1, Op_AddL, -1, -1) ||
4055 bmil.match(Op_XorL, -1, Op_AddL, -1, -1);
4056 }
4057 }
4058 return false;
4059 }
4060
4061 // Should the matcher clone input 'm' of node 'n'?
4062 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
4063 // If 'n' and 'm' are part of a graph for BMI instruction, clone the input 'm'.
4064 if (VM_Version::supports_bmi1() && VM_Version::supports_avx() && is_bmi_pattern(n, m)) {
4065 mstack.push(m, Visit);
4066 return true;
4067 }
4068 if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
4069 mstack.push(m, Visit); // m = ShiftCntV
4070 return true;
4071 }
4072 if (is_encode_and_store_pattern(n, m)) {
4073 mstack.push(m, Visit);
4074 return true;
4075 }
4076 return false;
4077 }
4078
4079 // Should the Matcher clone shifts on addressing modes, expecting them
4080 // to be subsumed into complex addressing expressions or compute them
4081 // into registers?
4082 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
4083 Node *off = m->in(AddPNode::Offset);
4084 if (off->is_Con()) {
4085 address_visited.test_set(m->_idx); // Flag as address_visited
4086 Node *adr = m->in(AddPNode::Address);
4087
4088 // Intel can handle 2 adds in addressing mode, with one of them using an immediate offset.
4089 // AtomicAdd is not an addressing expression.
4090 // Cheap to find it by looking for screwy base.
4091 if (adr->is_AddP() &&
4092 !adr->in(AddPNode::Base)->is_top() &&
4093 !adr->in(AddPNode::Offset)->is_Con() &&
4094 off->get_long() == (int) (off->get_long()) && // immL32
4095 // Are there other uses besides address expressions?
4096 !is_visited(adr)) {
4097 address_visited.set(adr->_idx); // Flag as address_visited
4098 Node *shift = adr->in(AddPNode::Offset);
4099 if (!clone_shift(shift, this, mstack, address_visited)) {
4100 mstack.push(shift, Pre_Visit);
4101 }
4102 mstack.push(adr->in(AddPNode::Address), Pre_Visit);
4103 mstack.push(adr->in(AddPNode::Base), Pre_Visit);
4104 } else {
4105 mstack.push(adr, Pre_Visit);
4106 }
4107
4108 // Clone X+offset as it also folds into most addressing expressions
4109 mstack.push(off, Visit);
4110 mstack.push(m->in(AddPNode::Base), Pre_Visit);
4111 return true;
4112 } else if (clone_shift(off, this, mstack, address_visited)) {
4113 address_visited.test_set(m->_idx); // Flag as address_visited
4114 mstack.push(m->in(AddPNode::Address), Pre_Visit);
4115 mstack.push(m->in(AddPNode::Base), Pre_Visit);
4116 return true;
4117 }
4118 return false;
4119 }
4120
4121 static inline Assembler::ComparisonPredicate booltest_pred_to_comparison_pred(int bt) {
4122 switch (bt) {
4123 case BoolTest::eq:
4124 return Assembler::eq;
4125 case BoolTest::ne:
4126 return Assembler::neq;
4127 case BoolTest::le:
4128 case BoolTest::ule:
4129 return Assembler::le;
4130 case BoolTest::ge:
4131 case BoolTest::uge:
4132 return Assembler::nlt;
4133 case BoolTest::lt:
4134 case BoolTest::ult:
4135 return Assembler::lt;
4136 case BoolTest::gt:
4137 case BoolTest::ugt:
4138 return Assembler::nle;
4139 default : ShouldNotReachHere(); return Assembler::_false;
4140 }
4141 }
4142
4143 static inline Assembler::ComparisonPredicateFP booltest_pred_to_comparison_pred_fp(int bt) {
4144 switch (bt) {
4145 case BoolTest::eq: return Assembler::EQ_OQ; // ordered non-signaling
4146 // As per JLS 15.21.1, != of NaNs is true. Thus use unordered compare.
4147 case BoolTest::ne: return Assembler::NEQ_UQ; // unordered non-signaling
4148 case BoolTest::le: return Assembler::LE_OQ; // ordered non-signaling
4149 case BoolTest::ge: return Assembler::GE_OQ; // ordered non-signaling
4150 case BoolTest::lt: return Assembler::LT_OQ; // ordered non-signaling
4151 case BoolTest::gt: return Assembler::GT_OQ; // ordered non-signaling
4152 default: ShouldNotReachHere(); return Assembler::FALSE_OS;
4153 }
4154 }
4155
4156 // Helper methods for MachSpillCopyNode::implementation().
4157 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
4158 int src_hi, int dst_hi, uint ireg, outputStream* st) {
4159 assert(ireg == Op_VecS || // 32bit vector
4160 ((src_lo & 1) == 0 && (src_lo + 1) == src_hi &&
4161 (dst_lo & 1) == 0 && (dst_lo + 1) == dst_hi),
4162 "no non-adjacent vector moves" );
4163 if (masm) {
4164 switch (ireg) {
4165 case Op_VecS: // copy whole register
4166 case Op_VecD:
4167 case Op_VecX:
4168 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4169 __ movdqu(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]));
4170 } else {
4171 __ vextractf32x4(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]), 0x0);
4172 }
4173 break;
4174 case Op_VecY:
4175 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4176 __ vmovdqu(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]));
4177 } else {
4178 __ vextractf64x4(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]), 0x0);
4179 }
4180 break;
4181 case Op_VecZ:
4182 __ evmovdquq(as_XMMRegister(Matcher::_regEncode[dst_lo]), as_XMMRegister(Matcher::_regEncode[src_lo]), 2);
4183 break;
4184 default:
4185 ShouldNotReachHere();
4186 }
4187 #ifndef PRODUCT
4188 } else {
4189 switch (ireg) {
4190 case Op_VecS:
4191 case Op_VecD:
4192 case Op_VecX:
4193 st->print("movdqu %s,%s\t# spill",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
4194 break;
4195 case Op_VecY:
4196 case Op_VecZ:
4197 st->print("vmovdqu %s,%s\t# spill",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
4198 break;
4199 default:
4200 ShouldNotReachHere();
4201 }
4202 #endif
4203 }
4204 }
4205
4206 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
4207 int stack_offset, int reg, uint ireg, outputStream* st) {
4208 if (masm) {
4209 if (is_load) {
4210 switch (ireg) {
4211 case Op_VecS:
4212 __ movdl(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4213 break;
4214 case Op_VecD:
4215 __ movq(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4216 break;
4217 case Op_VecX:
4218 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4219 __ movdqu(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4220 } else {
4221 __ vpxor(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), 2);
4222 __ vinsertf32x4(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset),0x0);
4223 }
4224 break;
4225 case Op_VecY:
4226 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4227 __ vmovdqu(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset));
4228 } else {
4229 __ vpxor(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), 2);
4230 __ vinsertf64x4(as_XMMRegister(Matcher::_regEncode[reg]), as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset),0x0);
4231 }
4232 break;
4233 case Op_VecZ:
4234 __ evmovdquq(as_XMMRegister(Matcher::_regEncode[reg]), Address(rsp, stack_offset), 2);
4235 break;
4236 default:
4237 ShouldNotReachHere();
4238 }
4239 } else { // store
4240 switch (ireg) {
4241 case Op_VecS:
4242 __ movdl(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4243 break;
4244 case Op_VecD:
4245 __ movq(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4246 break;
4247 case Op_VecX:
4248 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4249 __ movdqu(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4250 }
4251 else {
4252 __ vextractf32x4(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]), 0x0);
4253 }
4254 break;
4255 case Op_VecY:
4256 if ((UseAVX < 3) || VM_Version::supports_avx512vl()) {
4257 __ vmovdqu(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]));
4258 }
4259 else {
4260 __ vextractf64x4(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]), 0x0);
4261 }
4262 break;
4263 case Op_VecZ:
4264 __ evmovdquq(Address(rsp, stack_offset), as_XMMRegister(Matcher::_regEncode[reg]), 2);
4265 break;
4266 default:
4267 ShouldNotReachHere();
4268 }
4269 }
4270 #ifndef PRODUCT
4271 } else {
4272 if (is_load) {
4273 switch (ireg) {
4274 case Op_VecS:
4275 st->print("movd %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4276 break;
4277 case Op_VecD:
4278 st->print("movq %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4279 break;
4280 case Op_VecX:
4281 st->print("movdqu %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4282 break;
4283 case Op_VecY:
4284 case Op_VecZ:
4285 st->print("vmovdqu %s,[rsp + %d]\t# spill", Matcher::regName[reg], stack_offset);
4286 break;
4287 default:
4288 ShouldNotReachHere();
4289 }
4290 } else { // store
4291 switch (ireg) {
4292 case Op_VecS:
4293 st->print("movd [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4294 break;
4295 case Op_VecD:
4296 st->print("movq [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4297 break;
4298 case Op_VecX:
4299 st->print("movdqu [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4300 break;
4301 case Op_VecY:
4302 case Op_VecZ:
4303 st->print("vmovdqu [rsp + %d],%s\t# spill", stack_offset, Matcher::regName[reg]);
4304 break;
4305 default:
4306 ShouldNotReachHere();
4307 }
4308 }
4309 #endif
4310 }
4311 }
4312
4313 template <class T>
4314 static inline GrowableArray<jbyte>* vreplicate_imm(BasicType bt, T con, int len) {
4315 int size = type2aelembytes(bt) * len;
4316 GrowableArray<jbyte>* val = new GrowableArray<jbyte>(size, size, 0);
4317 for (int i = 0; i < len; i++) {
4318 int offset = i * type2aelembytes(bt);
4319 switch (bt) {
4320 case T_BYTE: val->at(i) = con; break;
4321 case T_SHORT: {
4322 jshort c = con;
4323 memcpy(val->adr_at(offset), &c, sizeof(jshort));
4324 break;
4325 }
4326 case T_INT: {
4327 jint c = con;
4328 memcpy(val->adr_at(offset), &c, sizeof(jint));
4329 break;
4330 }
4331 case T_LONG: {
4332 jlong c = con;
4333 memcpy(val->adr_at(offset), &c, sizeof(jlong));
4334 break;
4335 }
4336 case T_FLOAT: {
4337 jfloat c = con;
4338 memcpy(val->adr_at(offset), &c, sizeof(jfloat));
4339 break;
4340 }
4341 case T_DOUBLE: {
4342 jdouble c = con;
4343 memcpy(val->adr_at(offset), &c, sizeof(jdouble));
4344 break;
4345 }
4346 default: assert(false, "%s", type2name(bt));
4347 }
4348 }
4349 return val;
4350 }
4351
4352 static inline jlong high_bit_set(BasicType bt) {
4353 switch (bt) {
4354 case T_BYTE: return 0x8080808080808080;
4355 case T_SHORT: return 0x8000800080008000;
4356 case T_INT: return 0x8000000080000000;
4357 case T_LONG: return 0x8000000000000000;
4358 default:
4359 ShouldNotReachHere();
4360 return 0;
4361 }
4362 }
4363
4364 #ifndef PRODUCT
4365 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
4366 st->print("nop \t# %d bytes pad for loops and calls", _count);
4367 }
4368 #endif
4369
4370 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
4371 __ nop(_count);
4372 }
4373
4374 uint MachNopNode::size(PhaseRegAlloc*) const {
4375 return _count;
4376 }
4377
4378 #ifndef PRODUCT
4379 void MachBreakpointNode::format(PhaseRegAlloc*, outputStream* st) const {
4380 st->print("# breakpoint");
4381 }
4382 #endif
4383
4384 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const {
4385 __ int3();
4386 }
4387
4388 uint MachBreakpointNode::size(PhaseRegAlloc* ra_) const {
4389 return MachNode::size(ra_);
4390 }
4391
4392 %}
4393
4394 //----------ENCODING BLOCK-----------------------------------------------------
4395 // This block specifies the encoding classes used by the compiler to
4396 // output byte streams. Encoding classes are parameterized macros
4397 // used by Machine Instruction Nodes in order to generate the bit
4398 // encoding of the instruction. Operands specify their base encoding
4399 // interface with the interface keyword. There are currently
4400 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
4401 // COND_INTER. REG_INTER causes an operand to generate a function
4402 // which returns its register number when queried. CONST_INTER causes
4403 // an operand to generate a function which returns the value of the
4404 // constant when queried. MEMORY_INTER causes an operand to generate
4405 // four functions which return the Base Register, the Index Register,
4406 // the Scale Value, and the Offset Value of the operand when queried.
4407 // COND_INTER causes an operand to generate six functions which return
4408 // the encoding code (ie - encoding bits for the instruction)
4409 // associated with each basic boolean condition for a conditional
4410 // instruction.
4411 //
4412 // Instructions specify two basic values for encoding. Again, a
4413 // function is available to check if the constant displacement is an
4414 // oop. They use the ins_encode keyword to specify their encoding
4415 // classes (which must be a sequence of enc_class names, and their
4416 // parameters, specified in the encoding block), and they use the
4417 // opcode keyword to specify, in order, their primary, secondary, and
4418 // tertiary opcode. Only the opcode sections which a particular
4419 // instruction needs for encoding need to be specified.
4420 encode %{
4421 enc_class cdql_enc(no_rax_rdx_RegI div)
4422 %{
4423 // Full implementation of Java idiv and irem; checks for
4424 // special case as described in JVM spec., p.243 & p.271.
4425 //
4426 // normal case special case
4427 //
4428 // input : rax: dividend min_int
4429 // reg: divisor -1
4430 //
4431 // output: rax: quotient (= rax idiv reg) min_int
4432 // rdx: remainder (= rax irem reg) 0
4433 //
4434 // Code sequnce:
4435 //
4436 // 0: 3d 00 00 00 80 cmp $0x80000000,%eax
4437 // 5: 75 07/08 jne e <normal>
4438 // 7: 33 d2 xor %edx,%edx
4439 // [div >= 8 -> offset + 1]
4440 // [REX_B]
4441 // 9: 83 f9 ff cmp $0xffffffffffffffff,$div
4442 // c: 74 03/04 je 11 <done>
4443 // 000000000000000e <normal>:
4444 // e: 99 cltd
4445 // [div >= 8 -> offset + 1]
4446 // [REX_B]
4447 // f: f7 f9 idiv $div
4448 // 0000000000000011 <done>:
4449 Label normal;
4450 Label done;
4451
4452 // cmp $0x80000000,%eax
4453 __ cmpl(as_Register(RAX_enc), 0x80000000);
4454
4455 // jne e <normal>
4456 __ jccb(Assembler::notEqual, normal);
4457
4458 // xor %edx,%edx
4459 __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
4460
4461 // cmp $0xffffffffffffffff,%ecx
4462 __ cmpl($div$$Register, -1);
4463
4464 // je 11 <done>
4465 __ jccb(Assembler::equal, done);
4466
4467 // <normal>
4468 // cltd
4469 __ bind(normal);
4470 __ cdql();
4471
4472 // idivl
4473 // <done>
4474 __ idivl($div$$Register);
4475 __ bind(done);
4476 %}
4477
4478 enc_class cdqq_enc(no_rax_rdx_RegL div)
4479 %{
4480 // Full implementation of Java ldiv and lrem; checks for
4481 // special case as described in JVM spec., p.243 & p.271.
4482 //
4483 // normal case special case
4484 //
4485 // input : rax: dividend min_long
4486 // reg: divisor -1
4487 //
4488 // output: rax: quotient (= rax idiv reg) min_long
4489 // rdx: remainder (= rax irem reg) 0
4490 //
4491 // Code sequnce:
4492 //
4493 // 0: 48 ba 00 00 00 00 00 mov $0x8000000000000000,%rdx
4494 // 7: 00 00 80
4495 // a: 48 39 d0 cmp %rdx,%rax
4496 // d: 75 08 jne 17 <normal>
4497 // f: 33 d2 xor %edx,%edx
4498 // 11: 48 83 f9 ff cmp $0xffffffffffffffff,$div
4499 // 15: 74 05 je 1c <done>
4500 // 0000000000000017 <normal>:
4501 // 17: 48 99 cqto
4502 // 19: 48 f7 f9 idiv $div
4503 // 000000000000001c <done>:
4504 Label normal;
4505 Label done;
4506
4507 // mov $0x8000000000000000,%rdx
4508 __ mov64(as_Register(RDX_enc), 0x8000000000000000);
4509
4510 // cmp %rdx,%rax
4511 __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
4512
4513 // jne 17 <normal>
4514 __ jccb(Assembler::notEqual, normal);
4515
4516 // xor %edx,%edx
4517 __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
4518
4519 // cmp $0xffffffffffffffff,$div
4520 __ cmpq($div$$Register, -1);
4521
4522 // je 1e <done>
4523 __ jccb(Assembler::equal, done);
4524
4525 // <normal>
4526 // cqto
4527 __ bind(normal);
4528 __ cdqq();
4529
4530 // idivq (note: must be emitted by the user of this rule)
4531 // <done>
4532 __ idivq($div$$Register);
4533 __ bind(done);
4534 %}
4535
4536 enc_class clear_avx %{
4537 DEBUG_ONLY(int off0 = __ offset());
4538 if (generate_vzeroupper(Compile::current())) {
4539 // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
4540 // Clear upper bits of YMM registers when current compiled code uses
4541 // wide vectors to avoid AVX <-> SSE transition penalty during call.
4542 __ vzeroupper();
4543 }
4544 DEBUG_ONLY(int off1 = __ offset());
4545 assert(off1 - off0 == clear_avx_size(), "correct size prediction");
4546 %}
4547
4548 enc_class Java_To_Runtime(method meth) %{
4549 __ lea(r10, RuntimeAddress((address)$meth$$method));
4550 __ call(r10);
4551 __ post_call_nop();
4552 %}
4553
4554 enc_class Java_Static_Call(method meth)
4555 %{
4556 // JAVA STATIC CALL
4557 // CALL to fixup routine. Fixup routine uses ScopeDesc info to
4558 // determine who we intended to call.
4559 if (!_method) {
4560 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
4561 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
4562 // The NOP here is purely to ensure that eliding a call to
4563 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
4564 __ nop(5);
4565 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
4566 } else {
4567 int method_index = resolved_method_index(masm);
4568 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
4569 : static_call_Relocation::spec(method_index);
4570 address mark = __ pc();
4571 int call_offset = __ offset();
4572 __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
4573 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
4574 // Calls of the same statically bound method can share
4575 // a stub to the interpreter.
4576 __ code()->shared_stub_to_interp_for(_method, call_offset);
4577 } else {
4578 // Emit stubs for static call.
4579 address stub = CompiledDirectCall::emit_to_interp_stub(masm, mark);
4580 __ clear_inst_mark();
4581 if (stub == nullptr) {
4582 ciEnv::current()->record_failure("CodeCache is full");
4583 return;
4584 }
4585 }
4586 }
4587 __ post_call_nop();
4588 %}
4589
4590 enc_class Java_Dynamic_Call(method meth) %{
4591 __ ic_call((address)$meth$$method, resolved_method_index(masm));
4592 __ post_call_nop();
4593 %}
4594
4595 enc_class call_epilog %{
4596 if (VerifyStackAtCalls) {
4597 // Check that stack depth is unchanged: find majik cookie on stack
4598 int framesize = ra_->reg2offset_unchecked(OptoReg::add(ra_->_matcher._old_SP, -3*VMRegImpl::slots_per_word));
4599 Label L;
4600 __ cmpptr(Address(rsp, framesize), (int32_t)0xbadb100d);
4601 __ jccb(Assembler::equal, L);
4602 // Die if stack mismatch
4603 __ int3();
4604 __ bind(L);
4605 }
4606 %}
4607
4608 %}
4609
4610 //----------FRAME--------------------------------------------------------------
4611 // Definition of frame structure and management information.
4612 //
4613 // S T A C K L A Y O U T Allocators stack-slot number
4614 // | (to get allocators register number
4615 // G Owned by | | v add OptoReg::stack0())
4616 // r CALLER | |
4617 // o | +--------+ pad to even-align allocators stack-slot
4618 // w V | pad0 | numbers; owned by CALLER
4619 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
4620 // h ^ | in | 5
4621 // | | args | 4 Holes in incoming args owned by SELF
4622 // | | | | 3
4623 // | | +--------+
4624 // V | | old out| Empty on Intel, window on Sparc
4625 // | old |preserve| Must be even aligned.
4626 // | SP-+--------+----> Matcher::_old_SP, even aligned
4627 // | | in | 3 area for Intel ret address
4628 // Owned by |preserve| Empty on Sparc.
4629 // SELF +--------+
4630 // | | pad2 | 2 pad to align old SP
4631 // | +--------+ 1
4632 // | | locks | 0
4633 // | +--------+----> OptoReg::stack0(), even aligned
4634 // | | pad1 | 11 pad to align new SP
4635 // | +--------+
4636 // | | | 10
4637 // | | spills | 9 spills
4638 // V | | 8 (pad0 slot for callee)
4639 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
4640 // ^ | out | 7
4641 // | | args | 6 Holes in outgoing args owned by CALLEE
4642 // Owned by +--------+
4643 // CALLEE | new out| 6 Empty on Intel, window on Sparc
4644 // | new |preserve| Must be even-aligned.
4645 // | SP-+--------+----> Matcher::_new_SP, even aligned
4646 // | | |
4647 //
4648 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
4649 // known from SELF's arguments and the Java calling convention.
4650 // Region 6-7 is determined per call site.
4651 // Note 2: If the calling convention leaves holes in the incoming argument
4652 // area, those holes are owned by SELF. Holes in the outgoing area
4653 // are owned by the CALLEE. Holes should not be necessary in the
4654 // incoming area, as the Java calling convention is completely under
4655 // the control of the AD file. Doubles can be sorted and packed to
4656 // avoid holes. Holes in the outgoing arguments may be necessary for
4657 // varargs C calling conventions.
4658 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
4659 // even aligned with pad0 as needed.
4660 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
4661 // region 6-11 is even aligned; it may be padded out more so that
4662 // the region from SP to FP meets the minimum stack alignment.
4663 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
4664 // alignment. Region 11, pad1, may be dynamically extended so that
4665 // SP meets the minimum alignment.
4666
4667 frame
4668 %{
4669 // These three registers define part of the calling convention
4670 // between compiled code and the interpreter.
4671 inline_cache_reg(RAX); // Inline Cache Register
4672
4673 // Optional: name the operand used by cisc-spilling to access
4674 // [stack_pointer + offset]
4675 cisc_spilling_operand_name(indOffset32);
4676
4677 // Number of stack slots consumed by locking an object
4678 sync_stack_slots(2);
4679
4680 // Compiled code's Frame Pointer
4681 frame_pointer(RSP);
4682
4683 // Stack alignment requirement
4684 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
4685
4686 // Number of outgoing stack slots killed above the out_preserve_stack_slots
4687 // for calls to C. Supports the var-args backing area for register parms.
4688 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
4689
4690 // The after-PROLOG location of the return address. Location of
4691 // return address specifies a type (REG or STACK) and a number
4692 // representing the register number (i.e. - use a register name) or
4693 // stack slot.
4694 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4695 // Otherwise, it is above the locks and verification slot and alignment word
4696 return_addr(STACK - 2 +
4697 align_up((Compile::current()->in_preserve_stack_slots() +
4698 Compile::current()->fixed_slots()),
4699 stack_alignment_in_slots()));
4700
4701 // Location of compiled Java return values. Same as C for now.
4702 return_value
4703 %{
4704 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4705 "only return normal values");
4706
4707 static const int lo[Op_RegL + 1] = {
4708 0,
4709 0,
4710 RAX_num, // Op_RegN
4711 RAX_num, // Op_RegI
4712 RAX_num, // Op_RegP
4713 XMM0_num, // Op_RegF
4714 XMM0_num, // Op_RegD
4715 RAX_num // Op_RegL
4716 };
4717 static const int hi[Op_RegL + 1] = {
4718 0,
4719 0,
4720 OptoReg::Bad, // Op_RegN
4721 OptoReg::Bad, // Op_RegI
4722 RAX_H_num, // Op_RegP
4723 OptoReg::Bad, // Op_RegF
4724 XMM0b_num, // Op_RegD
4725 RAX_H_num // Op_RegL
4726 };
4727 // Excluded flags and vector registers.
4728 assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
4729 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4730 %}
4731 %}
4732
4733 //----------ATTRIBUTES---------------------------------------------------------
4734 //----------Operand Attributes-------------------------------------------------
4735 op_attrib op_cost(0); // Required cost attribute
4736
4737 //----------Instruction Attributes---------------------------------------------
4738 ins_attrib ins_cost(100); // Required cost attribute
4739 ins_attrib ins_size(8); // Required size attribute (in bits)
4740 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4741 // a non-matching short branch variant
4742 // of some long branch?
4743 ins_attrib ins_alignment(1); // Required alignment attribute (must
4744 // be a power of 2) specifies the
4745 // alignment that some part of the
4746 // instruction (not necessarily the
4747 // start) requires. If > 1, a
4748 // compute_padding() function must be
4749 // provided for the instruction
4750
4751 // Whether this node is expanded during code emission into a sequence of
4752 // instructions and the first instruction can perform an implicit null check.
4753 ins_attrib ins_is_late_expanded_null_check_candidate(false);
4754
4755 //----------OPERANDS-----------------------------------------------------------
4756 // Operand definitions must precede instruction definitions for correct parsing
4757 // in the ADLC because operands constitute user defined types which are used in
4758 // instruction definitions.
4759
4760 //----------Simple Operands----------------------------------------------------
4761 // Immediate Operands
4762 // Integer Immediate
4763 operand immI()
4764 %{
4765 match(ConI);
4766
4767 op_cost(10);
4768 format %{ %}
4769 interface(CONST_INTER);
4770 %}
4771
4772 // Constant for test vs zero
4773 operand immI_0()
4774 %{
4775 predicate(n->get_int() == 0);
4776 match(ConI);
4777
4778 op_cost(0);
4779 format %{ %}
4780 interface(CONST_INTER);
4781 %}
4782
4783 // Constant for increment
4784 operand immI_1()
4785 %{
4786 predicate(n->get_int() == 1);
4787 match(ConI);
4788
4789 op_cost(0);
4790 format %{ %}
4791 interface(CONST_INTER);
4792 %}
4793
4794 // Constant for decrement
4795 operand immI_M1()
4796 %{
4797 predicate(n->get_int() == -1);
4798 match(ConI);
4799
4800 op_cost(0);
4801 format %{ %}
4802 interface(CONST_INTER);
4803 %}
4804
4805 operand immI_2()
4806 %{
4807 predicate(n->get_int() == 2);
4808 match(ConI);
4809
4810 op_cost(0);
4811 format %{ %}
4812 interface(CONST_INTER);
4813 %}
4814
4815 operand immI_4()
4816 %{
4817 predicate(n->get_int() == 4);
4818 match(ConI);
4819
4820 op_cost(0);
4821 format %{ %}
4822 interface(CONST_INTER);
4823 %}
4824
4825 operand immI_8()
4826 %{
4827 predicate(n->get_int() == 8);
4828 match(ConI);
4829
4830 op_cost(0);
4831 format %{ %}
4832 interface(CONST_INTER);
4833 %}
4834
4835 // Valid scale values for addressing modes
4836 operand immI2()
4837 %{
4838 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4839 match(ConI);
4840
4841 format %{ %}
4842 interface(CONST_INTER);
4843 %}
4844
4845 operand immU7()
4846 %{
4847 predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
4848 match(ConI);
4849
4850 op_cost(5);
4851 format %{ %}
4852 interface(CONST_INTER);
4853 %}
4854
4855 operand immI8()
4856 %{
4857 predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
4858 match(ConI);
4859
4860 op_cost(5);
4861 format %{ %}
4862 interface(CONST_INTER);
4863 %}
4864
4865 operand immU8()
4866 %{
4867 predicate((0 <= n->get_int()) && (n->get_int() <= 255));
4868 match(ConI);
4869
4870 op_cost(5);
4871 format %{ %}
4872 interface(CONST_INTER);
4873 %}
4874
4875 operand immI16()
4876 %{
4877 predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
4878 match(ConI);
4879
4880 op_cost(10);
4881 format %{ %}
4882 interface(CONST_INTER);
4883 %}
4884
4885 // Int Immediate non-negative
4886 operand immU31()
4887 %{
4888 predicate(n->get_int() >= 0);
4889 match(ConI);
4890
4891 op_cost(0);
4892 format %{ %}
4893 interface(CONST_INTER);
4894 %}
4895
4896 // Pointer Immediate
4897 operand immP()
4898 %{
4899 match(ConP);
4900
4901 op_cost(10);
4902 format %{ %}
4903 interface(CONST_INTER);
4904 %}
4905
4906 // Null Pointer Immediate
4907 operand immP0()
4908 %{
4909 predicate(n->get_ptr() == 0);
4910 match(ConP);
4911
4912 op_cost(5);
4913 format %{ %}
4914 interface(CONST_INTER);
4915 %}
4916
4917 // Pointer Immediate
4918 operand immN() %{
4919 match(ConN);
4920
4921 op_cost(10);
4922 format %{ %}
4923 interface(CONST_INTER);
4924 %}
4925
4926 operand immNKlass() %{
4927 match(ConNKlass);
4928
4929 op_cost(10);
4930 format %{ %}
4931 interface(CONST_INTER);
4932 %}
4933
4934 // Null Pointer Immediate
4935 operand immN0() %{
4936 predicate(n->get_narrowcon() == 0);
4937 match(ConN);
4938
4939 op_cost(5);
4940 format %{ %}
4941 interface(CONST_INTER);
4942 %}
4943
4944 operand immP31()
4945 %{
4946 predicate(n->as_Type()->type()->is_ptr()->reloc() == relocInfo::none
4947 && (n->get_ptr() >> 31) == 0);
4948 match(ConP);
4949
4950 op_cost(5);
4951 format %{ %}
4952 interface(CONST_INTER);
4953 %}
4954
4955
4956 // Long Immediate
4957 operand immL()
4958 %{
4959 match(ConL);
4960
4961 op_cost(20);
4962 format %{ %}
4963 interface(CONST_INTER);
4964 %}
4965
4966 // Long Immediate 8-bit
4967 operand immL8()
4968 %{
4969 predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
4970 match(ConL);
4971
4972 op_cost(5);
4973 format %{ %}
4974 interface(CONST_INTER);
4975 %}
4976
4977 // Long Immediate 32-bit unsigned
4978 operand immUL32()
4979 %{
4980 predicate(n->get_long() == (unsigned int) (n->get_long()));
4981 match(ConL);
4982
4983 op_cost(10);
4984 format %{ %}
4985 interface(CONST_INTER);
4986 %}
4987
4988 // Long Immediate 32-bit signed
4989 operand immL32()
4990 %{
4991 predicate(n->get_long() == (int) (n->get_long()));
4992 match(ConL);
4993
4994 op_cost(15);
4995 format %{ %}
4996 interface(CONST_INTER);
4997 %}
4998
4999 operand immL_Pow2()
5000 %{
5001 predicate(is_power_of_2((julong)n->get_long()));
5002 match(ConL);
5003
5004 op_cost(15);
5005 format %{ %}
5006 interface(CONST_INTER);
5007 %}
5008
5009 operand immL_NotPow2()
5010 %{
5011 predicate(is_power_of_2((julong)~n->get_long()));
5012 match(ConL);
5013
5014 op_cost(15);
5015 format %{ %}
5016 interface(CONST_INTER);
5017 %}
5018
5019 // Long Immediate zero
5020 operand immL0()
5021 %{
5022 predicate(n->get_long() == 0L);
5023 match(ConL);
5024
5025 op_cost(10);
5026 format %{ %}
5027 interface(CONST_INTER);
5028 %}
5029
5030 // Constant for increment
5031 operand immL1()
5032 %{
5033 predicate(n->get_long() == 1);
5034 match(ConL);
5035
5036 format %{ %}
5037 interface(CONST_INTER);
5038 %}
5039
5040 // Constant for decrement
5041 operand immL_M1()
5042 %{
5043 predicate(n->get_long() == -1);
5044 match(ConL);
5045
5046 format %{ %}
5047 interface(CONST_INTER);
5048 %}
5049
5050 // Long Immediate: low 32-bit mask
5051 operand immL_32bits()
5052 %{
5053 predicate(n->get_long() == 0xFFFFFFFFL);
5054 match(ConL);
5055 op_cost(20);
5056
5057 format %{ %}
5058 interface(CONST_INTER);
5059 %}
5060
5061 // Int Immediate: 2^n-1, positive
5062 operand immI_Pow2M1()
5063 %{
5064 predicate((n->get_int() > 0)
5065 && is_power_of_2((juint)n->get_int() + 1));
5066 match(ConI);
5067
5068 op_cost(20);
5069 format %{ %}
5070 interface(CONST_INTER);
5071 %}
5072
5073 // Float Immediate zero
5074 operand immF0()
5075 %{
5076 predicate(jint_cast(n->getf()) == 0);
5077 match(ConF);
5078
5079 op_cost(5);
5080 format %{ %}
5081 interface(CONST_INTER);
5082 %}
5083
5084 // Float Immediate
5085 operand immF()
5086 %{
5087 match(ConF);
5088
5089 op_cost(15);
5090 format %{ %}
5091 interface(CONST_INTER);
5092 %}
5093
5094 // Half Float Immediate
5095 operand immH()
5096 %{
5097 match(ConH);
5098
5099 op_cost(15);
5100 format %{ %}
5101 interface(CONST_INTER);
5102 %}
5103
5104 // Double Immediate zero
5105 operand immD0()
5106 %{
5107 predicate(jlong_cast(n->getd()) == 0);
5108 match(ConD);
5109
5110 op_cost(5);
5111 format %{ %}
5112 interface(CONST_INTER);
5113 %}
5114
5115 // Double Immediate
5116 operand immD()
5117 %{
5118 match(ConD);
5119
5120 op_cost(15);
5121 format %{ %}
5122 interface(CONST_INTER);
5123 %}
5124
5125 // Immediates for special shifts (sign extend)
5126
5127 // Constants for increment
5128 operand immI_16()
5129 %{
5130 predicate(n->get_int() == 16);
5131 match(ConI);
5132
5133 format %{ %}
5134 interface(CONST_INTER);
5135 %}
5136
5137 operand immI_24()
5138 %{
5139 predicate(n->get_int() == 24);
5140 match(ConI);
5141
5142 format %{ %}
5143 interface(CONST_INTER);
5144 %}
5145
5146 // Constant for byte-wide masking
5147 operand immI_255()
5148 %{
5149 predicate(n->get_int() == 255);
5150 match(ConI);
5151
5152 format %{ %}
5153 interface(CONST_INTER);
5154 %}
5155
5156 // Constant for short-wide masking
5157 operand immI_65535()
5158 %{
5159 predicate(n->get_int() == 65535);
5160 match(ConI);
5161
5162 format %{ %}
5163 interface(CONST_INTER);
5164 %}
5165
5166 // Constant for byte-wide masking
5167 operand immL_255()
5168 %{
5169 predicate(n->get_long() == 255);
5170 match(ConL);
5171
5172 format %{ %}
5173 interface(CONST_INTER);
5174 %}
5175
5176 // Constant for short-wide masking
5177 operand immL_65535()
5178 %{
5179 predicate(n->get_long() == 65535);
5180 match(ConL);
5181
5182 format %{ %}
5183 interface(CONST_INTER);
5184 %}
5185
5186 // AOT Runtime Constants Address
5187 operand immAOTRuntimeConstantsAddress()
5188 %{
5189 // Check if the address is in the range of AOT Runtime Constants
5190 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
5191 match(ConP);
5192
5193 op_cost(0);
5194 format %{ %}
5195 interface(CONST_INTER);
5196 %}
5197
5198 operand kReg()
5199 %{
5200 constraint(ALLOC_IN_RC(vectmask_reg));
5201 match(RegVectMask);
5202 format %{%}
5203 interface(REG_INTER);
5204 %}
5205
5206 // Register Operands
5207 // Integer Register
5208 operand rRegI()
5209 %{
5210 constraint(ALLOC_IN_RC(int_reg));
5211 match(RegI);
5212
5213 match(rax_RegI);
5214 match(rbx_RegI);
5215 match(rcx_RegI);
5216 match(rdx_RegI);
5217 match(rdi_RegI);
5218
5219 format %{ %}
5220 interface(REG_INTER);
5221 %}
5222
5223 // Special Registers
5224 operand rax_RegI()
5225 %{
5226 constraint(ALLOC_IN_RC(int_rax_reg));
5227 match(RegI);
5228 match(rRegI);
5229
5230 format %{ "RAX" %}
5231 interface(REG_INTER);
5232 %}
5233
5234 // Special Registers
5235 operand rbx_RegI()
5236 %{
5237 constraint(ALLOC_IN_RC(int_rbx_reg));
5238 match(RegI);
5239 match(rRegI);
5240
5241 format %{ "RBX" %}
5242 interface(REG_INTER);
5243 %}
5244
5245 operand rcx_RegI()
5246 %{
5247 constraint(ALLOC_IN_RC(int_rcx_reg));
5248 match(RegI);
5249 match(rRegI);
5250
5251 format %{ "RCX" %}
5252 interface(REG_INTER);
5253 %}
5254
5255 operand rdx_RegI()
5256 %{
5257 constraint(ALLOC_IN_RC(int_rdx_reg));
5258 match(RegI);
5259 match(rRegI);
5260
5261 format %{ "RDX" %}
5262 interface(REG_INTER);
5263 %}
5264
5265 operand rdi_RegI()
5266 %{
5267 constraint(ALLOC_IN_RC(int_rdi_reg));
5268 match(RegI);
5269 match(rRegI);
5270
5271 format %{ "RDI" %}
5272 interface(REG_INTER);
5273 %}
5274
5275 operand no_rax_rdx_RegI()
5276 %{
5277 constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
5278 match(RegI);
5279 match(rbx_RegI);
5280 match(rcx_RegI);
5281 match(rdi_RegI);
5282
5283 format %{ %}
5284 interface(REG_INTER);
5285 %}
5286
5287 operand no_rbp_r13_RegI()
5288 %{
5289 constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
5290 match(RegI);
5291 match(rRegI);
5292 match(rax_RegI);
5293 match(rbx_RegI);
5294 match(rcx_RegI);
5295 match(rdx_RegI);
5296 match(rdi_RegI);
5297
5298 format %{ %}
5299 interface(REG_INTER);
5300 %}
5301
5302 // Pointer Register
5303 operand any_RegP()
5304 %{
5305 constraint(ALLOC_IN_RC(any_reg));
5306 match(RegP);
5307 match(rax_RegP);
5308 match(rbx_RegP);
5309 match(rdi_RegP);
5310 match(rsi_RegP);
5311 match(rbp_RegP);
5312 match(r15_RegP);
5313 match(rRegP);
5314
5315 format %{ %}
5316 interface(REG_INTER);
5317 %}
5318
5319 operand rRegP()
5320 %{
5321 constraint(ALLOC_IN_RC(ptr_reg));
5322 match(RegP);
5323 match(rax_RegP);
5324 match(rbx_RegP);
5325 match(rdi_RegP);
5326 match(rsi_RegP);
5327 match(rbp_RegP); // See Q&A below about
5328 match(r15_RegP); // r15_RegP and rbp_RegP.
5329
5330 format %{ %}
5331 interface(REG_INTER);
5332 %}
5333
5334 operand rRegN() %{
5335 constraint(ALLOC_IN_RC(int_reg));
5336 match(RegN);
5337
5338 format %{ %}
5339 interface(REG_INTER);
5340 %}
5341
5342 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
5343 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
5344 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
5345 // The output of an instruction is controlled by the allocator, which respects
5346 // register class masks, not match rules. Unless an instruction mentions
5347 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
5348 // by the allocator as an input.
5349 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
5350 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
5351 // result, RBP is not included in the output of the instruction either.
5352
5353 // This operand is not allowed to use RBP even if
5354 // RBP is not used to hold the frame pointer.
5355 operand no_rbp_RegP()
5356 %{
5357 constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
5358 match(RegP);
5359 match(rbx_RegP);
5360 match(rsi_RegP);
5361 match(rdi_RegP);
5362
5363 format %{ %}
5364 interface(REG_INTER);
5365 %}
5366
5367 // Special Registers
5368 // Return a pointer value
5369 operand rax_RegP()
5370 %{
5371 constraint(ALLOC_IN_RC(ptr_rax_reg));
5372 match(RegP);
5373 match(rRegP);
5374
5375 format %{ %}
5376 interface(REG_INTER);
5377 %}
5378
5379 // Special Registers
5380 // Return a compressed pointer value
5381 operand rax_RegN()
5382 %{
5383 constraint(ALLOC_IN_RC(int_rax_reg));
5384 match(RegN);
5385 match(rRegN);
5386
5387 format %{ %}
5388 interface(REG_INTER);
5389 %}
5390
5391 // Used in AtomicAdd
5392 operand rbx_RegP()
5393 %{
5394 constraint(ALLOC_IN_RC(ptr_rbx_reg));
5395 match(RegP);
5396 match(rRegP);
5397
5398 format %{ %}
5399 interface(REG_INTER);
5400 %}
5401
5402 operand rsi_RegP()
5403 %{
5404 constraint(ALLOC_IN_RC(ptr_rsi_reg));
5405 match(RegP);
5406 match(rRegP);
5407
5408 format %{ %}
5409 interface(REG_INTER);
5410 %}
5411
5412 operand rbp_RegP()
5413 %{
5414 constraint(ALLOC_IN_RC(ptr_rbp_reg));
5415 match(RegP);
5416 match(rRegP);
5417
5418 format %{ %}
5419 interface(REG_INTER);
5420 %}
5421
5422 // Used in rep stosq
5423 operand rdi_RegP()
5424 %{
5425 constraint(ALLOC_IN_RC(ptr_rdi_reg));
5426 match(RegP);
5427 match(rRegP);
5428
5429 format %{ %}
5430 interface(REG_INTER);
5431 %}
5432
5433 operand r15_RegP()
5434 %{
5435 constraint(ALLOC_IN_RC(ptr_r15_reg));
5436 match(RegP);
5437 match(rRegP);
5438
5439 format %{ %}
5440 interface(REG_INTER);
5441 %}
5442
5443 operand rRegL()
5444 %{
5445 constraint(ALLOC_IN_RC(long_reg));
5446 match(RegL);
5447 match(rax_RegL);
5448 match(rdx_RegL);
5449
5450 format %{ %}
5451 interface(REG_INTER);
5452 %}
5453
5454 // Special Registers
5455 operand no_rax_rdx_RegL()
5456 %{
5457 constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
5458 match(RegL);
5459 match(rRegL);
5460
5461 format %{ %}
5462 interface(REG_INTER);
5463 %}
5464
5465 operand rax_RegL()
5466 %{
5467 constraint(ALLOC_IN_RC(long_rax_reg));
5468 match(RegL);
5469 match(rRegL);
5470
5471 format %{ "RAX" %}
5472 interface(REG_INTER);
5473 %}
5474
5475 operand rcx_RegL()
5476 %{
5477 constraint(ALLOC_IN_RC(long_rcx_reg));
5478 match(RegL);
5479 match(rRegL);
5480
5481 format %{ %}
5482 interface(REG_INTER);
5483 %}
5484
5485 operand rdx_RegL()
5486 %{
5487 constraint(ALLOC_IN_RC(long_rdx_reg));
5488 match(RegL);
5489 match(rRegL);
5490
5491 format %{ %}
5492 interface(REG_INTER);
5493 %}
5494
5495 operand r11_RegL()
5496 %{
5497 constraint(ALLOC_IN_RC(long_r11_reg));
5498 match(RegL);
5499 match(rRegL);
5500
5501 format %{ %}
5502 interface(REG_INTER);
5503 %}
5504
5505 operand no_rbp_r13_RegL()
5506 %{
5507 constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
5508 match(RegL);
5509 match(rRegL);
5510 match(rax_RegL);
5511 match(rcx_RegL);
5512 match(rdx_RegL);
5513
5514 format %{ %}
5515 interface(REG_INTER);
5516 %}
5517
5518 // Flags register, used as output of compare instructions
5519 operand rFlagsReg()
5520 %{
5521 constraint(ALLOC_IN_RC(int_flags));
5522 match(RegFlags);
5523
5524 format %{ "RFLAGS" %}
5525 interface(REG_INTER);
5526 %}
5527
5528 // Flags register, used as output of FLOATING POINT compare instructions
5529 operand rFlagsRegU()
5530 %{
5531 constraint(ALLOC_IN_RC(int_flags));
5532 match(RegFlags);
5533
5534 format %{ "RFLAGS_U" %}
5535 interface(REG_INTER);
5536 %}
5537
5538 operand rFlagsRegUCF() %{
5539 constraint(ALLOC_IN_RC(int_flags));
5540 match(RegFlags);
5541 predicate(!UseAPX || !VM_Version::supports_avx10_2());
5542
5543 format %{ "RFLAGS_U_CF" %}
5544 interface(REG_INTER);
5545 %}
5546
5547 operand rFlagsRegUCFE() %{
5548 constraint(ALLOC_IN_RC(int_flags));
5549 match(RegFlags);
5550 predicate(UseAPX && VM_Version::supports_avx10_2());
5551
5552 format %{ "RFLAGS_U_CFE" %}
5553 interface(REG_INTER);
5554 %}
5555
5556 // Float register operands
5557 operand regF() %{
5558 constraint(ALLOC_IN_RC(float_reg));
5559 match(RegF);
5560
5561 format %{ %}
5562 interface(REG_INTER);
5563 %}
5564
5565 // Float register operands
5566 operand legRegF() %{
5567 constraint(ALLOC_IN_RC(float_reg_legacy));
5568 match(RegF);
5569
5570 format %{ %}
5571 interface(REG_INTER);
5572 %}
5573
5574 // Float register operands
5575 operand vlRegF() %{
5576 constraint(ALLOC_IN_RC(float_reg_vl));
5577 match(RegF);
5578
5579 format %{ %}
5580 interface(REG_INTER);
5581 %}
5582
5583 // Double register operands
5584 operand regD() %{
5585 constraint(ALLOC_IN_RC(double_reg));
5586 match(RegD);
5587
5588 format %{ %}
5589 interface(REG_INTER);
5590 %}
5591
5592 // Double register operands
5593 operand legRegD() %{
5594 constraint(ALLOC_IN_RC(double_reg_legacy));
5595 match(RegD);
5596
5597 format %{ %}
5598 interface(REG_INTER);
5599 %}
5600
5601 // Double register operands
5602 operand vlRegD() %{
5603 constraint(ALLOC_IN_RC(double_reg_vl));
5604 match(RegD);
5605
5606 format %{ %}
5607 interface(REG_INTER);
5608 %}
5609
5610 //----------Memory Operands----------------------------------------------------
5611 // Direct Memory Operand
5612 // operand direct(immP addr)
5613 // %{
5614 // match(addr);
5615
5616 // format %{ "[$addr]" %}
5617 // interface(MEMORY_INTER) %{
5618 // base(0xFFFFFFFF);
5619 // index(0x4);
5620 // scale(0x0);
5621 // disp($addr);
5622 // %}
5623 // %}
5624
5625 // Indirect Memory Operand
5626 operand indirect(any_RegP reg)
5627 %{
5628 constraint(ALLOC_IN_RC(ptr_reg));
5629 match(reg);
5630
5631 format %{ "[$reg]" %}
5632 interface(MEMORY_INTER) %{
5633 base($reg);
5634 index(0x4);
5635 scale(0x0);
5636 disp(0x0);
5637 %}
5638 %}
5639
5640 // Indirect Memory Plus Short Offset Operand
5641 operand indOffset8(any_RegP reg, immL8 off)
5642 %{
5643 constraint(ALLOC_IN_RC(ptr_reg));
5644 match(AddP reg off);
5645
5646 format %{ "[$reg + $off (8-bit)]" %}
5647 interface(MEMORY_INTER) %{
5648 base($reg);
5649 index(0x4);
5650 scale(0x0);
5651 disp($off);
5652 %}
5653 %}
5654
5655 // Indirect Memory Plus Long Offset Operand
5656 operand indOffset32(any_RegP reg, immL32 off)
5657 %{
5658 constraint(ALLOC_IN_RC(ptr_reg));
5659 match(AddP reg off);
5660
5661 format %{ "[$reg + $off (32-bit)]" %}
5662 interface(MEMORY_INTER) %{
5663 base($reg);
5664 index(0x4);
5665 scale(0x0);
5666 disp($off);
5667 %}
5668 %}
5669
5670 // Indirect Memory Plus Index Register Plus Offset Operand
5671 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
5672 %{
5673 constraint(ALLOC_IN_RC(ptr_reg));
5674 match(AddP (AddP reg lreg) off);
5675
5676 op_cost(10);
5677 format %{"[$reg + $off + $lreg]" %}
5678 interface(MEMORY_INTER) %{
5679 base($reg);
5680 index($lreg);
5681 scale(0x0);
5682 disp($off);
5683 %}
5684 %}
5685
5686 // Indirect Memory Plus Index Register Plus Offset Operand
5687 operand indIndex(any_RegP reg, rRegL lreg)
5688 %{
5689 constraint(ALLOC_IN_RC(ptr_reg));
5690 match(AddP reg lreg);
5691
5692 op_cost(10);
5693 format %{"[$reg + $lreg]" %}
5694 interface(MEMORY_INTER) %{
5695 base($reg);
5696 index($lreg);
5697 scale(0x0);
5698 disp(0x0);
5699 %}
5700 %}
5701
5702 // Indirect Memory Times Scale Plus Index Register
5703 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
5704 %{
5705 constraint(ALLOC_IN_RC(ptr_reg));
5706 match(AddP reg (LShiftL lreg scale));
5707
5708 op_cost(10);
5709 format %{"[$reg + $lreg << $scale]" %}
5710 interface(MEMORY_INTER) %{
5711 base($reg);
5712 index($lreg);
5713 scale($scale);
5714 disp(0x0);
5715 %}
5716 %}
5717
5718 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
5719 %{
5720 constraint(ALLOC_IN_RC(ptr_reg));
5721 predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
5722 match(AddP reg (LShiftL (ConvI2L idx) scale));
5723
5724 op_cost(10);
5725 format %{"[$reg + pos $idx << $scale]" %}
5726 interface(MEMORY_INTER) %{
5727 base($reg);
5728 index($idx);
5729 scale($scale);
5730 disp(0x0);
5731 %}
5732 %}
5733
5734 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
5735 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
5736 %{
5737 constraint(ALLOC_IN_RC(ptr_reg));
5738 match(AddP (AddP reg (LShiftL lreg scale)) off);
5739
5740 op_cost(10);
5741 format %{"[$reg + $off + $lreg << $scale]" %}
5742 interface(MEMORY_INTER) %{
5743 base($reg);
5744 index($lreg);
5745 scale($scale);
5746 disp($off);
5747 %}
5748 %}
5749
5750 // Indirect Memory Plus Positive Index Register Plus Offset Operand
5751 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
5752 %{
5753 constraint(ALLOC_IN_RC(ptr_reg));
5754 predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
5755 match(AddP (AddP reg (ConvI2L idx)) off);
5756
5757 op_cost(10);
5758 format %{"[$reg + $off + $idx]" %}
5759 interface(MEMORY_INTER) %{
5760 base($reg);
5761 index($idx);
5762 scale(0x0);
5763 disp($off);
5764 %}
5765 %}
5766
5767 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
5768 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
5769 %{
5770 constraint(ALLOC_IN_RC(ptr_reg));
5771 predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
5772 match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
5773
5774 op_cost(10);
5775 format %{"[$reg + $off + $idx << $scale]" %}
5776 interface(MEMORY_INTER) %{
5777 base($reg);
5778 index($idx);
5779 scale($scale);
5780 disp($off);
5781 %}
5782 %}
5783
5784 // Indirect Narrow Oop Plus Offset Operand
5785 // Note: x86 architecture doesn't support "scale * index + offset" without a base
5786 // we can't free r12 even with CompressedOops::base() == nullptr.
5787 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
5788 predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
5789 constraint(ALLOC_IN_RC(ptr_reg));
5790 match(AddP (DecodeN reg) off);
5791
5792 op_cost(10);
5793 format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
5794 interface(MEMORY_INTER) %{
5795 base(0xc); // R12
5796 index($reg);
5797 scale(0x3);
5798 disp($off);
5799 %}
5800 %}
5801
5802 // Indirect Memory Operand
5803 operand indirectNarrow(rRegN reg)
5804 %{
5805 predicate(CompressedOops::shift() == 0);
5806 constraint(ALLOC_IN_RC(ptr_reg));
5807 match(DecodeN reg);
5808
5809 format %{ "[$reg]" %}
5810 interface(MEMORY_INTER) %{
5811 base($reg);
5812 index(0x4);
5813 scale(0x0);
5814 disp(0x0);
5815 %}
5816 %}
5817
5818 // Indirect Memory Plus Short Offset Operand
5819 operand indOffset8Narrow(rRegN reg, immL8 off)
5820 %{
5821 predicate(CompressedOops::shift() == 0);
5822 constraint(ALLOC_IN_RC(ptr_reg));
5823 match(AddP (DecodeN reg) off);
5824
5825 format %{ "[$reg + $off (8-bit)]" %}
5826 interface(MEMORY_INTER) %{
5827 base($reg);
5828 index(0x4);
5829 scale(0x0);
5830 disp($off);
5831 %}
5832 %}
5833
5834 // Indirect Memory Plus Long Offset Operand
5835 operand indOffset32Narrow(rRegN reg, immL32 off)
5836 %{
5837 predicate(CompressedOops::shift() == 0);
5838 constraint(ALLOC_IN_RC(ptr_reg));
5839 match(AddP (DecodeN reg) off);
5840
5841 format %{ "[$reg + $off (32-bit)]" %}
5842 interface(MEMORY_INTER) %{
5843 base($reg);
5844 index(0x4);
5845 scale(0x0);
5846 disp($off);
5847 %}
5848 %}
5849
5850 // Indirect Memory Plus Index Register Plus Offset Operand
5851 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
5852 %{
5853 predicate(CompressedOops::shift() == 0);
5854 constraint(ALLOC_IN_RC(ptr_reg));
5855 match(AddP (AddP (DecodeN reg) lreg) off);
5856
5857 op_cost(10);
5858 format %{"[$reg + $off + $lreg]" %}
5859 interface(MEMORY_INTER) %{
5860 base($reg);
5861 index($lreg);
5862 scale(0x0);
5863 disp($off);
5864 %}
5865 %}
5866
5867 // Indirect Memory Plus Index Register Plus Offset Operand
5868 operand indIndexNarrow(rRegN reg, rRegL lreg)
5869 %{
5870 predicate(CompressedOops::shift() == 0);
5871 constraint(ALLOC_IN_RC(ptr_reg));
5872 match(AddP (DecodeN reg) lreg);
5873
5874 op_cost(10);
5875 format %{"[$reg + $lreg]" %}
5876 interface(MEMORY_INTER) %{
5877 base($reg);
5878 index($lreg);
5879 scale(0x0);
5880 disp(0x0);
5881 %}
5882 %}
5883
5884 // Indirect Memory Times Scale Plus Index Register
5885 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
5886 %{
5887 predicate(CompressedOops::shift() == 0);
5888 constraint(ALLOC_IN_RC(ptr_reg));
5889 match(AddP (DecodeN reg) (LShiftL lreg scale));
5890
5891 op_cost(10);
5892 format %{"[$reg + $lreg << $scale]" %}
5893 interface(MEMORY_INTER) %{
5894 base($reg);
5895 index($lreg);
5896 scale($scale);
5897 disp(0x0);
5898 %}
5899 %}
5900
5901 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
5902 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
5903 %{
5904 predicate(CompressedOops::shift() == 0);
5905 constraint(ALLOC_IN_RC(ptr_reg));
5906 match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
5907
5908 op_cost(10);
5909 format %{"[$reg + $off + $lreg << $scale]" %}
5910 interface(MEMORY_INTER) %{
5911 base($reg);
5912 index($lreg);
5913 scale($scale);
5914 disp($off);
5915 %}
5916 %}
5917
5918 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
5919 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
5920 %{
5921 constraint(ALLOC_IN_RC(ptr_reg));
5922 predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
5923 match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
5924
5925 op_cost(10);
5926 format %{"[$reg + $off + $idx]" %}
5927 interface(MEMORY_INTER) %{
5928 base($reg);
5929 index($idx);
5930 scale(0x0);
5931 disp($off);
5932 %}
5933 %}
5934
5935 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
5936 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
5937 %{
5938 constraint(ALLOC_IN_RC(ptr_reg));
5939 predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
5940 match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
5941
5942 op_cost(10);
5943 format %{"[$reg + $off + $idx << $scale]" %}
5944 interface(MEMORY_INTER) %{
5945 base($reg);
5946 index($idx);
5947 scale($scale);
5948 disp($off);
5949 %}
5950 %}
5951
5952 //----------Special Memory Operands--------------------------------------------
5953 // Stack Slot Operand - This operand is used for loading and storing temporary
5954 // values on the stack where a match requires a value to
5955 // flow through memory.
5956 operand stackSlotP(sRegP reg)
5957 %{
5958 constraint(ALLOC_IN_RC(stack_slots));
5959 // No match rule because this operand is only generated in matching
5960
5961 format %{ "[$reg]" %}
5962 interface(MEMORY_INTER) %{
5963 base(0x4); // RSP
5964 index(0x4); // No Index
5965 scale(0x0); // No Scale
5966 disp($reg); // Stack Offset
5967 %}
5968 %}
5969
5970 operand stackSlotI(sRegI reg)
5971 %{
5972 constraint(ALLOC_IN_RC(stack_slots));
5973 // No match rule because this operand is only generated in matching
5974
5975 format %{ "[$reg]" %}
5976 interface(MEMORY_INTER) %{
5977 base(0x4); // RSP
5978 index(0x4); // No Index
5979 scale(0x0); // No Scale
5980 disp($reg); // Stack Offset
5981 %}
5982 %}
5983
5984 operand stackSlotF(sRegF reg)
5985 %{
5986 constraint(ALLOC_IN_RC(stack_slots));
5987 // No match rule because this operand is only generated in matching
5988
5989 format %{ "[$reg]" %}
5990 interface(MEMORY_INTER) %{
5991 base(0x4); // RSP
5992 index(0x4); // No Index
5993 scale(0x0); // No Scale
5994 disp($reg); // Stack Offset
5995 %}
5996 %}
5997
5998 operand stackSlotD(sRegD reg)
5999 %{
6000 constraint(ALLOC_IN_RC(stack_slots));
6001 // No match rule because this operand is only generated in matching
6002
6003 format %{ "[$reg]" %}
6004 interface(MEMORY_INTER) %{
6005 base(0x4); // RSP
6006 index(0x4); // No Index
6007 scale(0x0); // No Scale
6008 disp($reg); // Stack Offset
6009 %}
6010 %}
6011 operand stackSlotL(sRegL reg)
6012 %{
6013 constraint(ALLOC_IN_RC(stack_slots));
6014 // No match rule because this operand is only generated in matching
6015
6016 format %{ "[$reg]" %}
6017 interface(MEMORY_INTER) %{
6018 base(0x4); // RSP
6019 index(0x4); // No Index
6020 scale(0x0); // No Scale
6021 disp($reg); // Stack Offset
6022 %}
6023 %}
6024
6025 //----------Conditional Branch Operands----------------------------------------
6026 // Comparison Op - This is the operation of the comparison, and is limited to
6027 // the following set of codes:
6028 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6029 //
6030 // Other attributes of the comparison, such as unsignedness, are specified
6031 // by the comparison instruction that sets a condition code flags register.
6032 // That result is represented by a flags operand whose subtype is appropriate
6033 // to the unsignedness (etc.) of the comparison.
6034 //
6035 // Later, the instruction which matches both the Comparison Op (a Bool) and
6036 // the flags (produced by the Cmp) specifies the coding of the comparison op
6037 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6038
6039 // Comparison Code
6040 operand cmpOp()
6041 %{
6042 match(Bool);
6043
6044 format %{ "" %}
6045 interface(COND_INTER) %{
6046 equal(0x4, "e");
6047 not_equal(0x5, "ne");
6048 less(0xc, "l");
6049 greater_equal(0xd, "ge");
6050 less_equal(0xe, "le");
6051 greater(0xf, "g");
6052 overflow(0x0, "o");
6053 no_overflow(0x1, "no");
6054 %}
6055 %}
6056
6057 // Comparison Code, unsigned compare. Used by FP also, with
6058 // C2 (unordered) turned into GT or LT already. The other bits
6059 // C0 and C3 are turned into Carry & Zero flags.
6060 operand cmpOpU()
6061 %{
6062 match(Bool);
6063
6064 format %{ "" %}
6065 interface(COND_INTER) %{
6066 equal(0x4, "e");
6067 not_equal(0x5, "ne");
6068 less(0x2, "b");
6069 greater_equal(0x3, "ae");
6070 less_equal(0x6, "be");
6071 greater(0x7, "a");
6072 overflow(0x0, "o");
6073 no_overflow(0x1, "no");
6074 %}
6075 %}
6076
6077
6078 // Floating comparisons that don't require any fixup for the unordered case,
6079 // If both inputs of the comparison are the same, ZF is always set so we
6080 // don't need to use cmpOpUCF2 for eq/ne
6081 operand cmpOpUCF() %{
6082 match(Bool);
6083 predicate((!UseAPX || !VM_Version::supports_avx10_2()) &&
6084 (n->as_Bool()->_test._test == BoolTest::lt ||
6085 n->as_Bool()->_test._test == BoolTest::ge ||
6086 n->as_Bool()->_test._test == BoolTest::le ||
6087 n->as_Bool()->_test._test == BoolTest::gt ||
6088 n->in(1)->in(1) == n->in(1)->in(2)));
6089 format %{ "" %}
6090 interface(COND_INTER) %{
6091 equal(0xb, "np");
6092 not_equal(0xa, "p");
6093 less(0x2, "b");
6094 greater_equal(0x3, "ae");
6095 less_equal(0x6, "be");
6096 greater(0x7, "a");
6097 overflow(0x0, "o");
6098 no_overflow(0x1, "no");
6099 %}
6100 %}
6101
6102
6103 // Floating comparisons that can be fixed up with extra conditional jumps
6104 operand cmpOpUCF2() %{
6105 match(Bool);
6106 predicate((!UseAPX || !VM_Version::supports_avx10_2()) &&
6107 (n->as_Bool()->_test._test == BoolTest::ne ||
6108 n->as_Bool()->_test._test == BoolTest::eq) &&
6109 n->in(1)->in(1) != n->in(1)->in(2));
6110 format %{ "" %}
6111 interface(COND_INTER) %{
6112 equal(0x4, "e");
6113 not_equal(0x5, "ne");
6114 less(0x2, "b");
6115 greater_equal(0x3, "ae");
6116 less_equal(0x6, "be");
6117 greater(0x7, "a");
6118 overflow(0x0, "o");
6119 no_overflow(0x1, "no");
6120 %}
6121 %}
6122
6123
6124 // Floating point comparisons that set condition flags to test more directly,
6125 // Unsigned tests are used for G (>) and GE (>=) conditions while signed tests
6126 // are used for L (<) and LE (<=) conditions. It's important to convert these
6127 // latter conditions to ones that use unsigned tests before passing into an
6128 // instruction because the preceding comparison might be based on a three way
6129 // comparison (CmpF3 or CmpD3) that also assigns unordered outcomes to -1.
6130 operand cmpOpUCFE()
6131 %{
6132 match(Bool);
6133 predicate((UseAPX && VM_Version::supports_avx10_2()) &&
6134 (n->as_Bool()->_test._test == BoolTest::ne ||
6135 n->as_Bool()->_test._test == BoolTest::eq ||
6136 n->as_Bool()->_test._test == BoolTest::lt ||
6137 n->as_Bool()->_test._test == BoolTest::ge ||
6138 n->as_Bool()->_test._test == BoolTest::le ||
6139 n->as_Bool()->_test._test == BoolTest::gt));
6140
6141 format %{ "" %}
6142 interface(COND_INTER) %{
6143 equal(0x4, "e");
6144 not_equal(0x5, "ne");
6145 less(0x2, "b");
6146 greater_equal(0x3, "ae");
6147 less_equal(0x6, "be");
6148 greater(0x7, "a");
6149 overflow(0x0, "o");
6150 no_overflow(0x1, "no");
6151 %}
6152 %}
6153
6154 // Operands for bound floating pointer register arguments
6155 operand rxmm0() %{
6156 constraint(ALLOC_IN_RC(xmm0_reg));
6157 match(VecX);
6158 format%{%}
6159 interface(REG_INTER);
6160 %}
6161
6162 // Vectors
6163
6164 // Dummy generic vector class. Should be used for all vector operands.
6165 // Replaced with vec[SDXYZ] during post-selection pass.
6166 operand vec() %{
6167 constraint(ALLOC_IN_RC(dynamic));
6168 match(VecX);
6169 match(VecY);
6170 match(VecZ);
6171 match(VecS);
6172 match(VecD);
6173
6174 format %{ %}
6175 interface(REG_INTER);
6176 %}
6177
6178 // Dummy generic legacy vector class. Should be used for all legacy vector operands.
6179 // Replaced with legVec[SDXYZ] during post-selection cleanup.
6180 // Note: legacy register class is used to avoid extra (unneeded in 32-bit VM)
6181 // runtime code generation via reg_class_dynamic.
6182 operand legVec() %{
6183 constraint(ALLOC_IN_RC(dynamic));
6184 match(VecX);
6185 match(VecY);
6186 match(VecZ);
6187 match(VecS);
6188 match(VecD);
6189
6190 format %{ %}
6191 interface(REG_INTER);
6192 %}
6193
6194 // Replaces vec during post-selection cleanup. See above.
6195 operand vecS() %{
6196 constraint(ALLOC_IN_RC(vectors_reg_vlbwdq));
6197 match(VecS);
6198
6199 format %{ %}
6200 interface(REG_INTER);
6201 %}
6202
6203 // Replaces legVec during post-selection cleanup. See above.
6204 operand legVecS() %{
6205 constraint(ALLOC_IN_RC(vectors_reg_legacy));
6206 match(VecS);
6207
6208 format %{ %}
6209 interface(REG_INTER);
6210 %}
6211
6212 // Replaces vec during post-selection cleanup. See above.
6213 operand vecD() %{
6214 constraint(ALLOC_IN_RC(vectord_reg_vlbwdq));
6215 match(VecD);
6216
6217 format %{ %}
6218 interface(REG_INTER);
6219 %}
6220
6221 // Replaces legVec during post-selection cleanup. See above.
6222 operand legVecD() %{
6223 constraint(ALLOC_IN_RC(vectord_reg_legacy));
6224 match(VecD);
6225
6226 format %{ %}
6227 interface(REG_INTER);
6228 %}
6229
6230 // Replaces vec during post-selection cleanup. See above.
6231 operand vecX() %{
6232 constraint(ALLOC_IN_RC(vectorx_reg_vlbwdq));
6233 match(VecX);
6234
6235 format %{ %}
6236 interface(REG_INTER);
6237 %}
6238
6239 // Replaces legVec during post-selection cleanup. See above.
6240 operand legVecX() %{
6241 constraint(ALLOC_IN_RC(vectorx_reg_legacy));
6242 match(VecX);
6243
6244 format %{ %}
6245 interface(REG_INTER);
6246 %}
6247
6248 // Replaces vec during post-selection cleanup. See above.
6249 operand vecY() %{
6250 constraint(ALLOC_IN_RC(vectory_reg_vlbwdq));
6251 match(VecY);
6252
6253 format %{ %}
6254 interface(REG_INTER);
6255 %}
6256
6257 // Replaces legVec during post-selection cleanup. See above.
6258 operand legVecY() %{
6259 constraint(ALLOC_IN_RC(vectory_reg_legacy));
6260 match(VecY);
6261
6262 format %{ %}
6263 interface(REG_INTER);
6264 %}
6265
6266 // Replaces vec during post-selection cleanup. See above.
6267 operand vecZ() %{
6268 constraint(ALLOC_IN_RC(vectorz_reg));
6269 match(VecZ);
6270
6271 format %{ %}
6272 interface(REG_INTER);
6273 %}
6274
6275 // Replaces legVec during post-selection cleanup. See above.
6276 operand legVecZ() %{
6277 constraint(ALLOC_IN_RC(vectorz_reg_legacy));
6278 match(VecZ);
6279
6280 format %{ %}
6281 interface(REG_INTER);
6282 %}
6283
6284 //----------OPERAND CLASSES----------------------------------------------------
6285 // Operand Classes are groups of operands that are used as to simplify
6286 // instruction definitions by not requiring the AD writer to specify separate
6287 // instructions for every form of operand when the instruction accepts
6288 // multiple operand types with the same basic encoding and format. The classic
6289 // case of this is memory operands.
6290
6291 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
6292 indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
6293 indCompressedOopOffset,
6294 indirectNarrow, indOffset8Narrow, indOffset32Narrow,
6295 indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
6296 indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
6297
6298 //----------PIPELINE-----------------------------------------------------------
6299 // Rules which define the behavior of the target architectures pipeline.
6300 pipeline %{
6301
6302 //----------ATTRIBUTES---------------------------------------------------------
6303 attributes %{
6304 variable_size_instructions; // Fixed size instructions
6305 max_instructions_per_bundle = 3; // Up to 3 instructions per bundle
6306 instruction_unit_size = 1; // An instruction is 1 bytes long
6307 instruction_fetch_unit_size = 16; // The processor fetches one line
6308 instruction_fetch_units = 1; // of 16 bytes
6309 %}
6310
6311 //----------RESOURCES----------------------------------------------------------
6312 // Resources are the functional units available to the machine
6313
6314 // Generic P2/P3 pipeline
6315 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
6316 // 3 instructions decoded per cycle.
6317 // 2 load/store ops per cycle, 1 branch, 1 FPU,
6318 // 3 ALU op, only ALU0 handles mul instructions.
6319 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
6320 MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
6321 BR, FPU,
6322 ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
6323
6324 //----------PIPELINE DESCRIPTION-----------------------------------------------
6325 // Pipeline Description specifies the stages in the machine's pipeline
6326
6327 // Generic P2/P3 pipeline
6328 pipe_desc(S0, S1, S2, S3, S4, S5);
6329
6330 //----------PIPELINE CLASSES---------------------------------------------------
6331 // Pipeline Classes describe the stages in which input and output are
6332 // referenced by the hardware pipeline.
6333
6334 // Naming convention: ialu or fpu
6335 // Then: _reg
6336 // Then: _reg if there is a 2nd register
6337 // Then: _long if it's a pair of instructions implementing a long
6338 // Then: _fat if it requires the big decoder
6339 // Or: _mem if it requires the big decoder and a memory unit.
6340
6341 // Integer ALU reg operation
6342 pipe_class ialu_reg(rRegI dst)
6343 %{
6344 single_instruction;
6345 dst : S4(write);
6346 dst : S3(read);
6347 DECODE : S0; // any decoder
6348 ALU : S3; // any alu
6349 %}
6350
6351 // Long ALU reg operation
6352 pipe_class ialu_reg_long(rRegL dst)
6353 %{
6354 instruction_count(2);
6355 dst : S4(write);
6356 dst : S3(read);
6357 DECODE : S0(2); // any 2 decoders
6358 ALU : S3(2); // both alus
6359 %}
6360
6361 // Integer ALU reg operation using big decoder
6362 pipe_class ialu_reg_fat(rRegI dst)
6363 %{
6364 single_instruction;
6365 dst : S4(write);
6366 dst : S3(read);
6367 D0 : S0; // big decoder only
6368 ALU : S3; // any alu
6369 %}
6370
6371 // Integer ALU reg-reg operation
6372 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
6373 %{
6374 single_instruction;
6375 dst : S4(write);
6376 src : S3(read);
6377 DECODE : S0; // any decoder
6378 ALU : S3; // any alu
6379 %}
6380
6381 // Integer ALU reg-reg operation
6382 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
6383 %{
6384 single_instruction;
6385 dst : S4(write);
6386 src : S3(read);
6387 D0 : S0; // big decoder only
6388 ALU : S3; // any alu
6389 %}
6390
6391 // Integer ALU reg-mem operation
6392 pipe_class ialu_reg_mem(rRegI dst, memory mem)
6393 %{
6394 single_instruction;
6395 dst : S5(write);
6396 mem : S3(read);
6397 D0 : S0; // big decoder only
6398 ALU : S4; // any alu
6399 MEM : S3; // any mem
6400 %}
6401
6402 // Integer mem operation (prefetch)
6403 pipe_class ialu_mem(memory mem)
6404 %{
6405 single_instruction;
6406 mem : S3(read);
6407 D0 : S0; // big decoder only
6408 MEM : S3; // any mem
6409 %}
6410
6411 // Integer Store to Memory
6412 pipe_class ialu_mem_reg(memory mem, rRegI src)
6413 %{
6414 single_instruction;
6415 mem : S3(read);
6416 src : S5(read);
6417 D0 : S0; // big decoder only
6418 ALU : S4; // any alu
6419 MEM : S3;
6420 %}
6421
6422 // // Long Store to Memory
6423 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
6424 // %{
6425 // instruction_count(2);
6426 // mem : S3(read);
6427 // src : S5(read);
6428 // D0 : S0(2); // big decoder only; twice
6429 // ALU : S4(2); // any 2 alus
6430 // MEM : S3(2); // Both mems
6431 // %}
6432
6433 // Integer Store to Memory
6434 pipe_class ialu_mem_imm(memory mem)
6435 %{
6436 single_instruction;
6437 mem : S3(read);
6438 D0 : S0; // big decoder only
6439 ALU : S4; // any alu
6440 MEM : S3;
6441 %}
6442
6443 // Integer ALU0 reg-reg operation
6444 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
6445 %{
6446 single_instruction;
6447 dst : S4(write);
6448 src : S3(read);
6449 D0 : S0; // Big decoder only
6450 ALU0 : S3; // only alu0
6451 %}
6452
6453 // Integer ALU0 reg-mem operation
6454 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
6455 %{
6456 single_instruction;
6457 dst : S5(write);
6458 mem : S3(read);
6459 D0 : S0; // big decoder only
6460 ALU0 : S4; // ALU0 only
6461 MEM : S3; // any mem
6462 %}
6463
6464 // Integer ALU reg-reg operation
6465 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
6466 %{
6467 single_instruction;
6468 cr : S4(write);
6469 src1 : S3(read);
6470 src2 : S3(read);
6471 DECODE : S0; // any decoder
6472 ALU : S3; // any alu
6473 %}
6474
6475 // Integer ALU reg-imm operation
6476 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
6477 %{
6478 single_instruction;
6479 cr : S4(write);
6480 src1 : S3(read);
6481 DECODE : S0; // any decoder
6482 ALU : S3; // any alu
6483 %}
6484
6485 // Integer ALU reg-mem operation
6486 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
6487 %{
6488 single_instruction;
6489 cr : S4(write);
6490 src1 : S3(read);
6491 src2 : S3(read);
6492 D0 : S0; // big decoder only
6493 ALU : S4; // any alu
6494 MEM : S3;
6495 %}
6496
6497 // Conditional move reg-reg
6498 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
6499 %{
6500 instruction_count(4);
6501 y : S4(read);
6502 q : S3(read);
6503 p : S3(read);
6504 DECODE : S0(4); // any decoder
6505 %}
6506
6507 // Conditional move reg-reg
6508 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
6509 %{
6510 single_instruction;
6511 dst : S4(write);
6512 src : S3(read);
6513 cr : S3(read);
6514 DECODE : S0; // any decoder
6515 %}
6516
6517 // Conditional move reg-mem
6518 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
6519 %{
6520 single_instruction;
6521 dst : S4(write);
6522 src : S3(read);
6523 cr : S3(read);
6524 DECODE : S0; // any decoder
6525 MEM : S3;
6526 %}
6527
6528 // Conditional move reg-reg long
6529 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
6530 %{
6531 single_instruction;
6532 dst : S4(write);
6533 src : S3(read);
6534 cr : S3(read);
6535 DECODE : S0(2); // any 2 decoders
6536 %}
6537
6538 // Float reg-reg operation
6539 pipe_class fpu_reg(regD dst)
6540 %{
6541 instruction_count(2);
6542 dst : S3(read);
6543 DECODE : S0(2); // any 2 decoders
6544 FPU : S3;
6545 %}
6546
6547 // Float reg-reg operation
6548 pipe_class fpu_reg_reg(regD dst, regD src)
6549 %{
6550 instruction_count(2);
6551 dst : S4(write);
6552 src : S3(read);
6553 DECODE : S0(2); // any 2 decoders
6554 FPU : S3;
6555 %}
6556
6557 // Float reg-reg operation
6558 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
6559 %{
6560 instruction_count(3);
6561 dst : S4(write);
6562 src1 : S3(read);
6563 src2 : S3(read);
6564 DECODE : S0(3); // any 3 decoders
6565 FPU : S3(2);
6566 %}
6567
6568 // Float reg-reg operation
6569 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
6570 %{
6571 instruction_count(4);
6572 dst : S4(write);
6573 src1 : S3(read);
6574 src2 : S3(read);
6575 src3 : S3(read);
6576 DECODE : S0(4); // any 3 decoders
6577 FPU : S3(2);
6578 %}
6579
6580 // Float reg-reg operation
6581 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
6582 %{
6583 instruction_count(4);
6584 dst : S4(write);
6585 src1 : S3(read);
6586 src2 : S3(read);
6587 src3 : S3(read);
6588 DECODE : S1(3); // any 3 decoders
6589 D0 : S0; // Big decoder only
6590 FPU : S3(2);
6591 MEM : S3;
6592 %}
6593
6594 // Float reg-mem operation
6595 pipe_class fpu_reg_mem(regD dst, memory mem)
6596 %{
6597 instruction_count(2);
6598 dst : S5(write);
6599 mem : S3(read);
6600 D0 : S0; // big decoder only
6601 DECODE : S1; // any decoder for FPU POP
6602 FPU : S4;
6603 MEM : S3; // any mem
6604 %}
6605
6606 // Float reg-mem operation
6607 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
6608 %{
6609 instruction_count(3);
6610 dst : S5(write);
6611 src1 : S3(read);
6612 mem : S3(read);
6613 D0 : S0; // big decoder only
6614 DECODE : S1(2); // any decoder for FPU POP
6615 FPU : S4;
6616 MEM : S3; // any mem
6617 %}
6618
6619 // Float mem-reg operation
6620 pipe_class fpu_mem_reg(memory mem, regD src)
6621 %{
6622 instruction_count(2);
6623 src : S5(read);
6624 mem : S3(read);
6625 DECODE : S0; // any decoder for FPU PUSH
6626 D0 : S1; // big decoder only
6627 FPU : S4;
6628 MEM : S3; // any mem
6629 %}
6630
6631 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
6632 %{
6633 instruction_count(3);
6634 src1 : S3(read);
6635 src2 : S3(read);
6636 mem : S3(read);
6637 DECODE : S0(2); // any decoder for FPU PUSH
6638 D0 : S1; // big decoder only
6639 FPU : S4;
6640 MEM : S3; // any mem
6641 %}
6642
6643 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
6644 %{
6645 instruction_count(3);
6646 src1 : S3(read);
6647 src2 : S3(read);
6648 mem : S4(read);
6649 DECODE : S0; // any decoder for FPU PUSH
6650 D0 : S0(2); // big decoder only
6651 FPU : S4;
6652 MEM : S3(2); // any mem
6653 %}
6654
6655 pipe_class fpu_mem_mem(memory dst, memory src1)
6656 %{
6657 instruction_count(2);
6658 src1 : S3(read);
6659 dst : S4(read);
6660 D0 : S0(2); // big decoder only
6661 MEM : S3(2); // any mem
6662 %}
6663
6664 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
6665 %{
6666 instruction_count(3);
6667 src1 : S3(read);
6668 src2 : S3(read);
6669 dst : S4(read);
6670 D0 : S0(3); // big decoder only
6671 FPU : S4;
6672 MEM : S3(3); // any mem
6673 %}
6674
6675 pipe_class fpu_mem_reg_con(memory mem, regD src1)
6676 %{
6677 instruction_count(3);
6678 src1 : S4(read);
6679 mem : S4(read);
6680 DECODE : S0; // any decoder for FPU PUSH
6681 D0 : S0(2); // big decoder only
6682 FPU : S4;
6683 MEM : S3(2); // any mem
6684 %}
6685
6686 // Float load constant
6687 pipe_class fpu_reg_con(regD dst)
6688 %{
6689 instruction_count(2);
6690 dst : S5(write);
6691 D0 : S0; // big decoder only for the load
6692 DECODE : S1; // any decoder for FPU POP
6693 FPU : S4;
6694 MEM : S3; // any mem
6695 %}
6696
6697 // Float load constant
6698 pipe_class fpu_reg_reg_con(regD dst, regD src)
6699 %{
6700 instruction_count(3);
6701 dst : S5(write);
6702 src : S3(read);
6703 D0 : S0; // big decoder only for the load
6704 DECODE : S1(2); // any decoder for FPU POP
6705 FPU : S4;
6706 MEM : S3; // any mem
6707 %}
6708
6709 // UnConditional branch
6710 pipe_class pipe_jmp(label labl)
6711 %{
6712 single_instruction;
6713 BR : S3;
6714 %}
6715
6716 // Conditional branch
6717 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
6718 %{
6719 single_instruction;
6720 cr : S1(read);
6721 BR : S3;
6722 %}
6723
6724 // Allocation idiom
6725 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
6726 %{
6727 instruction_count(1); force_serialization;
6728 fixed_latency(6);
6729 heap_ptr : S3(read);
6730 DECODE : S0(3);
6731 D0 : S2;
6732 MEM : S3;
6733 ALU : S3(2);
6734 dst : S5(write);
6735 BR : S5;
6736 %}
6737
6738 // Generic big/slow expanded idiom
6739 pipe_class pipe_slow()
6740 %{
6741 instruction_count(10); multiple_bundles; force_serialization;
6742 fixed_latency(100);
6743 D0 : S0(2);
6744 MEM : S3(2);
6745 %}
6746
6747 // The real do-nothing guy
6748 pipe_class empty()
6749 %{
6750 instruction_count(0);
6751 %}
6752
6753 // Define the class for the Nop node
6754 define
6755 %{
6756 MachNop = empty;
6757 %}
6758
6759 %}
6760
6761 //----------INSTRUCTIONS-------------------------------------------------------
6762 //
6763 // match -- States which machine-independent subtree may be replaced
6764 // by this instruction.
6765 // ins_cost -- The estimated cost of this instruction is used by instruction
6766 // selection to identify a minimum cost tree of machine
6767 // instructions that matches a tree of machine-independent
6768 // instructions.
6769 // format -- A string providing the disassembly for this instruction.
6770 // The value of an instruction's operand may be inserted
6771 // by referring to it with a '$' prefix.
6772 // opcode -- Three instruction opcodes may be provided. These are referred
6773 // to within an encode class as $primary, $secondary, and $tertiary
6774 // rrspectively. The primary opcode is commonly used to
6775 // indicate the type of machine instruction, while secondary
6776 // and tertiary are often used for prefix options or addressing
6777 // modes.
6778 // ins_encode -- A list of encode classes with parameters. The encode class
6779 // name must have been defined in an 'enc_class' specification
6780 // in the encode section of the architecture description.
6781
6782 // ============================================================================
6783
6784 instruct ShouldNotReachHere() %{
6785 match(Halt);
6786 format %{ "stop\t# ShouldNotReachHere" %}
6787 ins_encode %{
6788 if (is_reachable()) {
6789 const char* str = __ code_string(_halt_reason);
6790 __ stop(str);
6791 }
6792 %}
6793 ins_pipe(pipe_slow);
6794 %}
6795
6796 // ============================================================================
6797
6798 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
6799 // Load Float
6800 instruct MoveF2VL(vlRegF dst, regF src) %{
6801 match(Set dst src);
6802 format %{ "movss $dst,$src\t! load float (4 bytes)" %}
6803 ins_encode %{
6804 ShouldNotReachHere();
6805 %}
6806 ins_pipe( fpu_reg_reg );
6807 %}
6808
6809 // Load Float
6810 instruct MoveF2LEG(legRegF dst, regF src) %{
6811 match(Set dst src);
6812 format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
6813 ins_encode %{
6814 ShouldNotReachHere();
6815 %}
6816 ins_pipe( fpu_reg_reg );
6817 %}
6818
6819 // Load Float
6820 instruct MoveVL2F(regF dst, vlRegF src) %{
6821 match(Set dst src);
6822 format %{ "movss $dst,$src\t! load float (4 bytes)" %}
6823 ins_encode %{
6824 ShouldNotReachHere();
6825 %}
6826 ins_pipe( fpu_reg_reg );
6827 %}
6828
6829 // Load Float
6830 instruct MoveLEG2F(regF dst, legRegF src) %{
6831 match(Set dst src);
6832 format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
6833 ins_encode %{
6834 ShouldNotReachHere();
6835 %}
6836 ins_pipe( fpu_reg_reg );
6837 %}
6838
6839 // Load Double
6840 instruct MoveD2VL(vlRegD dst, regD src) %{
6841 match(Set dst src);
6842 format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
6843 ins_encode %{
6844 ShouldNotReachHere();
6845 %}
6846 ins_pipe( fpu_reg_reg );
6847 %}
6848
6849 // Load Double
6850 instruct MoveD2LEG(legRegD dst, regD src) %{
6851 match(Set dst src);
6852 format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
6853 ins_encode %{
6854 ShouldNotReachHere();
6855 %}
6856 ins_pipe( fpu_reg_reg );
6857 %}
6858
6859 // Load Double
6860 instruct MoveVL2D(regD dst, vlRegD src) %{
6861 match(Set dst src);
6862 format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
6863 ins_encode %{
6864 ShouldNotReachHere();
6865 %}
6866 ins_pipe( fpu_reg_reg );
6867 %}
6868
6869 // Load Double
6870 instruct MoveLEG2D(regD dst, legRegD src) %{
6871 match(Set dst src);
6872 format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
6873 ins_encode %{
6874 ShouldNotReachHere();
6875 %}
6876 ins_pipe( fpu_reg_reg );
6877 %}
6878
6879 //----------Load/Store/Move Instructions---------------------------------------
6880 //----------Load Instructions--------------------------------------------------
6881
6882 // Load Byte (8 bit signed)
6883 instruct loadB(rRegI dst, memory mem)
6884 %{
6885 match(Set dst (LoadB mem));
6886
6887 ins_cost(125);
6888 format %{ "movsbl $dst, $mem\t# byte" %}
6889
6890 ins_encode %{
6891 __ movsbl($dst$$Register, $mem$$Address);
6892 %}
6893
6894 ins_pipe(ialu_reg_mem);
6895 %}
6896
6897 // Load Byte (8 bit signed) into Long Register
6898 instruct loadB2L(rRegL dst, memory mem)
6899 %{
6900 match(Set dst (ConvI2L (LoadB mem)));
6901
6902 ins_cost(125);
6903 format %{ "movsbq $dst, $mem\t# byte -> long" %}
6904
6905 ins_encode %{
6906 __ movsbq($dst$$Register, $mem$$Address);
6907 %}
6908
6909 ins_pipe(ialu_reg_mem);
6910 %}
6911
6912 // Load Unsigned Byte (8 bit UNsigned)
6913 instruct loadUB(rRegI dst, memory mem)
6914 %{
6915 match(Set dst (LoadUB mem));
6916
6917 ins_cost(125);
6918 format %{ "movzbl $dst, $mem\t# ubyte" %}
6919
6920 ins_encode %{
6921 __ movzbl($dst$$Register, $mem$$Address);
6922 %}
6923
6924 ins_pipe(ialu_reg_mem);
6925 %}
6926
6927 // Load Unsigned Byte (8 bit UNsigned) into Long Register
6928 instruct loadUB2L(rRegL dst, memory mem)
6929 %{
6930 match(Set dst (ConvI2L (LoadUB mem)));
6931
6932 ins_cost(125);
6933 format %{ "movzbq $dst, $mem\t# ubyte -> long" %}
6934
6935 ins_encode %{
6936 __ movzbq($dst$$Register, $mem$$Address);
6937 %}
6938
6939 ins_pipe(ialu_reg_mem);
6940 %}
6941
6942 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
6943 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
6944 match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
6945 effect(KILL cr);
6946
6947 format %{ "movzbq $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
6948 "andl $dst, right_n_bits($mask, 8)" %}
6949 ins_encode %{
6950 Register Rdst = $dst$$Register;
6951 __ movzbq(Rdst, $mem$$Address);
6952 __ andl(Rdst, $mask$$constant & right_n_bits(8));
6953 %}
6954 ins_pipe(ialu_reg_mem);
6955 %}
6956
6957 // Load Short (16 bit signed)
6958 instruct loadS(rRegI dst, memory mem)
6959 %{
6960 match(Set dst (LoadS mem));
6961
6962 ins_cost(125);
6963 format %{ "movswl $dst, $mem\t# short" %}
6964
6965 ins_encode %{
6966 __ movswl($dst$$Register, $mem$$Address);
6967 %}
6968
6969 ins_pipe(ialu_reg_mem);
6970 %}
6971
6972 // Load Short (16 bit signed) to Byte (8 bit signed)
6973 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
6974 match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
6975
6976 ins_cost(125);
6977 format %{ "movsbl $dst, $mem\t# short -> byte" %}
6978 ins_encode %{
6979 __ movsbl($dst$$Register, $mem$$Address);
6980 %}
6981 ins_pipe(ialu_reg_mem);
6982 %}
6983
6984 // Load Short (16 bit signed) into Long Register
6985 instruct loadS2L(rRegL dst, memory mem)
6986 %{
6987 match(Set dst (ConvI2L (LoadS mem)));
6988
6989 ins_cost(125);
6990 format %{ "movswq $dst, $mem\t# short -> long" %}
6991
6992 ins_encode %{
6993 __ movswq($dst$$Register, $mem$$Address);
6994 %}
6995
6996 ins_pipe(ialu_reg_mem);
6997 %}
6998
6999 // Load Unsigned Short/Char (16 bit UNsigned)
7000 instruct loadUS(rRegI dst, memory mem)
7001 %{
7002 match(Set dst (LoadUS mem));
7003
7004 ins_cost(125);
7005 format %{ "movzwl $dst, $mem\t# ushort/char" %}
7006
7007 ins_encode %{
7008 __ movzwl($dst$$Register, $mem$$Address);
7009 %}
7010
7011 ins_pipe(ialu_reg_mem);
7012 %}
7013
7014 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
7015 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
7016 match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
7017
7018 ins_cost(125);
7019 format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
7020 ins_encode %{
7021 __ movsbl($dst$$Register, $mem$$Address);
7022 %}
7023 ins_pipe(ialu_reg_mem);
7024 %}
7025
7026 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
7027 instruct loadUS2L(rRegL dst, memory mem)
7028 %{
7029 match(Set dst (ConvI2L (LoadUS mem)));
7030
7031 ins_cost(125);
7032 format %{ "movzwq $dst, $mem\t# ushort/char -> long" %}
7033
7034 ins_encode %{
7035 __ movzwq($dst$$Register, $mem$$Address);
7036 %}
7037
7038 ins_pipe(ialu_reg_mem);
7039 %}
7040
7041 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
7042 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
7043 match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
7044
7045 format %{ "movzbq $dst, $mem\t# ushort/char & 0xFF -> long" %}
7046 ins_encode %{
7047 __ movzbq($dst$$Register, $mem$$Address);
7048 %}
7049 ins_pipe(ialu_reg_mem);
7050 %}
7051
7052 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
7053 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
7054 match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
7055 effect(KILL cr);
7056
7057 format %{ "movzwq $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
7058 "andl $dst, right_n_bits($mask, 16)" %}
7059 ins_encode %{
7060 Register Rdst = $dst$$Register;
7061 __ movzwq(Rdst, $mem$$Address);
7062 __ andl(Rdst, $mask$$constant & right_n_bits(16));
7063 %}
7064 ins_pipe(ialu_reg_mem);
7065 %}
7066
7067 // Load Integer
7068 instruct loadI(rRegI dst, memory mem)
7069 %{
7070 match(Set dst (LoadI mem));
7071
7072 ins_cost(125);
7073 format %{ "movl $dst, $mem\t# int" %}
7074
7075 ins_encode %{
7076 __ movl($dst$$Register, $mem$$Address);
7077 %}
7078
7079 ins_pipe(ialu_reg_mem);
7080 %}
7081
7082 // Load Integer (32 bit signed) to Byte (8 bit signed)
7083 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
7084 match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
7085
7086 ins_cost(125);
7087 format %{ "movsbl $dst, $mem\t# int -> byte" %}
7088 ins_encode %{
7089 __ movsbl($dst$$Register, $mem$$Address);
7090 %}
7091 ins_pipe(ialu_reg_mem);
7092 %}
7093
7094 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
7095 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
7096 match(Set dst (AndI (LoadI mem) mask));
7097
7098 ins_cost(125);
7099 format %{ "movzbl $dst, $mem\t# int -> ubyte" %}
7100 ins_encode %{
7101 __ movzbl($dst$$Register, $mem$$Address);
7102 %}
7103 ins_pipe(ialu_reg_mem);
7104 %}
7105
7106 // Load Integer (32 bit signed) to Short (16 bit signed)
7107 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
7108 match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
7109
7110 ins_cost(125);
7111 format %{ "movswl $dst, $mem\t# int -> short" %}
7112 ins_encode %{
7113 __ movswl($dst$$Register, $mem$$Address);
7114 %}
7115 ins_pipe(ialu_reg_mem);
7116 %}
7117
7118 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
7119 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
7120 match(Set dst (AndI (LoadI mem) mask));
7121
7122 ins_cost(125);
7123 format %{ "movzwl $dst, $mem\t# int -> ushort/char" %}
7124 ins_encode %{
7125 __ movzwl($dst$$Register, $mem$$Address);
7126 %}
7127 ins_pipe(ialu_reg_mem);
7128 %}
7129
7130 // Load Integer into Long Register
7131 instruct loadI2L(rRegL dst, memory mem)
7132 %{
7133 match(Set dst (ConvI2L (LoadI mem)));
7134
7135 ins_cost(125);
7136 format %{ "movslq $dst, $mem\t# int -> long" %}
7137
7138 ins_encode %{
7139 __ movslq($dst$$Register, $mem$$Address);
7140 %}
7141
7142 ins_pipe(ialu_reg_mem);
7143 %}
7144
7145 // Load Integer with mask 0xFF into Long Register
7146 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
7147 match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
7148
7149 format %{ "movzbq $dst, $mem\t# int & 0xFF -> long" %}
7150 ins_encode %{
7151 __ movzbq($dst$$Register, $mem$$Address);
7152 %}
7153 ins_pipe(ialu_reg_mem);
7154 %}
7155
7156 // Load Integer with mask 0xFFFF into Long Register
7157 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
7158 match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
7159
7160 format %{ "movzwq $dst, $mem\t# int & 0xFFFF -> long" %}
7161 ins_encode %{
7162 __ movzwq($dst$$Register, $mem$$Address);
7163 %}
7164 ins_pipe(ialu_reg_mem);
7165 %}
7166
7167 // Load Integer with a 31-bit mask into Long Register
7168 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
7169 match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
7170 effect(KILL cr);
7171
7172 format %{ "movl $dst, $mem\t# int & 31-bit mask -> long\n\t"
7173 "andl $dst, $mask" %}
7174 ins_encode %{
7175 Register Rdst = $dst$$Register;
7176 __ movl(Rdst, $mem$$Address);
7177 __ andl(Rdst, $mask$$constant);
7178 %}
7179 ins_pipe(ialu_reg_mem);
7180 %}
7181
7182 // Load Unsigned Integer into Long Register
7183 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
7184 %{
7185 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7186
7187 ins_cost(125);
7188 format %{ "movl $dst, $mem\t# uint -> long" %}
7189
7190 ins_encode %{
7191 __ movl($dst$$Register, $mem$$Address);
7192 %}
7193
7194 ins_pipe(ialu_reg_mem);
7195 %}
7196
7197 // Load Long
7198 instruct loadL(rRegL dst, memory mem)
7199 %{
7200 match(Set dst (LoadL mem));
7201
7202 ins_cost(125);
7203 format %{ "movq $dst, $mem\t# long" %}
7204
7205 ins_encode %{
7206 __ movq($dst$$Register, $mem$$Address);
7207 %}
7208
7209 ins_pipe(ialu_reg_mem); // XXX
7210 %}
7211
7212 // Load Range
7213 instruct loadRange(rRegI dst, memory mem)
7214 %{
7215 match(Set dst (LoadRange mem));
7216
7217 ins_cost(125); // XXX
7218 format %{ "movl $dst, $mem\t# range" %}
7219 ins_encode %{
7220 __ movl($dst$$Register, $mem$$Address);
7221 %}
7222 ins_pipe(ialu_reg_mem);
7223 %}
7224
7225 // Load Pointer
7226 instruct loadP(rRegP dst, memory mem)
7227 %{
7228 match(Set dst (LoadP mem));
7229 predicate(n->as_Load()->barrier_data() == 0);
7230
7231 ins_cost(125); // XXX
7232 format %{ "movq $dst, $mem\t# ptr" %}
7233 ins_encode %{
7234 __ movq($dst$$Register, $mem$$Address);
7235 %}
7236 ins_pipe(ialu_reg_mem); // XXX
7237 %}
7238
7239 // Load Compressed Pointer
7240 instruct loadN(rRegN dst, memory mem)
7241 %{
7242 predicate(n->as_Load()->barrier_data() == 0);
7243 match(Set dst (LoadN mem));
7244
7245 ins_cost(125); // XXX
7246 format %{ "movl $dst, $mem\t# compressed ptr" %}
7247 ins_encode %{
7248 __ movl($dst$$Register, $mem$$Address);
7249 %}
7250 ins_pipe(ialu_reg_mem); // XXX
7251 %}
7252
7253
7254 // Load Klass Pointer
7255 instruct loadKlass(rRegP dst, memory mem)
7256 %{
7257 match(Set dst (LoadKlass mem));
7258
7259 ins_cost(125); // XXX
7260 format %{ "movq $dst, $mem\t# class" %}
7261 ins_encode %{
7262 __ movq($dst$$Register, $mem$$Address);
7263 %}
7264 ins_pipe(ialu_reg_mem); // XXX
7265 %}
7266
7267 // Load narrow Klass Pointer
7268 instruct loadNKlass(rRegN dst, memory mem)
7269 %{
7270 predicate(!UseCompactObjectHeaders);
7271 match(Set dst (LoadNKlass mem));
7272
7273 ins_cost(125); // XXX
7274 format %{ "movl $dst, $mem\t# compressed klass ptr" %}
7275 ins_encode %{
7276 __ movl($dst$$Register, $mem$$Address);
7277 %}
7278 ins_pipe(ialu_reg_mem); // XXX
7279 %}
7280
7281 instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr)
7282 %{
7283 predicate(UseCompactObjectHeaders);
7284 match(Set dst (LoadNKlass mem));
7285 effect(KILL cr);
7286 ins_cost(125);
7287 format %{
7288 "movl $dst, $mem\t# compressed klass ptr, shifted\n\t"
7289 "shrl $dst, markWord::klass_shift_at_offset"
7290 %}
7291 ins_encode %{
7292 __ movl($dst$$Register, $mem$$Address);
7293 __ shrl($dst$$Register, markWord::klass_shift_at_offset);
7294 %}
7295 ins_pipe(ialu_reg_mem);
7296 %}
7297
7298 // Load Float
7299 instruct loadF(regF dst, memory mem)
7300 %{
7301 match(Set dst (LoadF mem));
7302
7303 ins_cost(145); // XXX
7304 format %{ "movss $dst, $mem\t# float" %}
7305 ins_encode %{
7306 __ movflt($dst$$XMMRegister, $mem$$Address);
7307 %}
7308 ins_pipe(pipe_slow); // XXX
7309 %}
7310
7311 // Load Double
7312 instruct loadD_partial(regD dst, memory mem)
7313 %{
7314 predicate(!UseXmmLoadAndClearUpper);
7315 match(Set dst (LoadD mem));
7316
7317 ins_cost(145); // XXX
7318 format %{ "movlpd $dst, $mem\t# double" %}
7319 ins_encode %{
7320 __ movdbl($dst$$XMMRegister, $mem$$Address);
7321 %}
7322 ins_pipe(pipe_slow); // XXX
7323 %}
7324
7325 instruct loadD(regD dst, memory mem)
7326 %{
7327 predicate(UseXmmLoadAndClearUpper);
7328 match(Set dst (LoadD mem));
7329
7330 ins_cost(145); // XXX
7331 format %{ "movsd $dst, $mem\t# double" %}
7332 ins_encode %{
7333 __ movdbl($dst$$XMMRegister, $mem$$Address);
7334 %}
7335 ins_pipe(pipe_slow); // XXX
7336 %}
7337
7338 instruct loadAOTRCAddress(rRegP dst, immAOTRuntimeConstantsAddress con)
7339 %{
7340 match(Set dst con);
7341
7342 format %{ "leaq $dst, $con\t# AOT Runtime Constants Address" %}
7343
7344 ins_encode %{
7345 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
7346 %}
7347
7348 ins_pipe(ialu_reg_fat);
7349 %}
7350
7351 // min = java.lang.Math.min(float a, float b)
7352 // max = java.lang.Math.max(float a, float b)
7353 instruct minmaxF_reg_avx10_2(regF dst, regF a, regF b)
7354 %{
7355 predicate(VM_Version::supports_avx10_2() && !VLoopReductions::is_reduction(n));
7356 match(Set dst (MaxF a b));
7357 match(Set dst (MinF a b));
7358
7359 format %{ "minmaxF $dst, $a, $b" %}
7360 ins_encode %{
7361 int opcode = this->ideal_Opcode();
7362 __ sminmax_fp_avx10_2(opcode, T_FLOAT, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister);
7363 %}
7364 ins_pipe( pipe_slow );
7365 %}
7366
7367 instruct minmaxF_reduction_reg_avx10_2(regF dst, regF a, regF b, rRegI rtmp, rFlagsReg cr)
7368 %{
7369 predicate(VM_Version::supports_avx10_2() && VLoopReductions::is_reduction(n));
7370 match(Set dst (MaxF a b));
7371 match(Set dst (MinF a b));
7372 effect(USE a, USE b, TEMP rtmp, KILL cr);
7373
7374 format %{ "minmaxF_reduction $dst, $a, $b \t! using $rtmp as TEMP" %}
7375 ins_encode %{
7376 int opcode = this->ideal_Opcode();
7377 bool min = (opcode == Op_MinF) ? true : false;
7378 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7379 min, fp_prec_flt /*pt*/);
7380 %}
7381 ins_pipe( pipe_slow );
7382 %}
7383
7384 // min = java.lang.Math.min(float a, float b)
7385 // max = java.lang.Math.max(float a, float b)
7386 instruct minmaxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp)
7387 %{
7388 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
7389 match(Set dst (MaxF a b));
7390 match(Set dst (MinF a b));
7391 effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
7392
7393 format %{ "minmaxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
7394 ins_encode %{
7395 int opcode = this->ideal_Opcode();
7396 int param_opcode = (opcode == Op_MinF) ? Op_MinV : Op_MaxV;
7397 __ vminmax_fp(param_opcode, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister,
7398 $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
7399 %}
7400 ins_pipe( pipe_slow );
7401 %}
7402
7403 instruct minmaxF_reduction_reg(legRegF dst, legRegF a, legRegF b, rRegI rtmp, rFlagsReg cr)
7404 %{
7405 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
7406 match(Set dst (MaxF a b));
7407 match(Set dst (MinF a b));
7408 effect(USE a, USE b, TEMP rtmp, KILL cr);
7409
7410 format %{ "minmaxF_reduction $dst, $a, $b \t!using $rtmp as TEMP" %}
7411 ins_encode %{
7412 int opcode = this->ideal_Opcode();
7413 bool min = (opcode == Op_MinF) ? true : false;
7414 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7415 min, fp_prec_flt /*pt*/);
7416 %}
7417 ins_pipe( pipe_slow );
7418 %}
7419
7420 // min = java.lang.Math.min(double a, double b)
7421 // max = java.lang.Math.max(double a, double b)
7422 instruct minmaxD_reg_avx10_2(regD dst, regD a, regD b)
7423 %{
7424 predicate(VM_Version::supports_avx10_2() && !VLoopReductions::is_reduction(n));
7425 match(Set dst (MaxD a b));
7426 match(Set dst (MinD a b));
7427
7428 format %{ "minmaxD $dst, $a, $b" %}
7429 ins_encode %{
7430 int opcode = this->ideal_Opcode();
7431 __ sminmax_fp_avx10_2(opcode, T_DOUBLE, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister);
7432 %}
7433 ins_pipe( pipe_slow );
7434 %}
7435
7436 instruct minmaxD_reduction_reg_avx10_2(regD dst, regD a, regD b, rRegI rtmp, rFlagsReg cr)
7437 %{
7438 predicate(VM_Version::supports_avx10_2() && VLoopReductions::is_reduction(n));
7439 match(Set dst (MaxD a b));
7440 match(Set dst (MinD a b));
7441 effect(USE a, USE b, TEMP rtmp, KILL cr);
7442
7443 format %{ "minmaxD_reduction $dst, $a, $b \t! using $rtmp as TEMP" %}
7444 ins_encode %{
7445 int opcode = this->ideal_Opcode();
7446 bool min = (opcode == Op_MinD) ? true : false;
7447 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7448 min, fp_prec_dbl /*pt*/);
7449 %}
7450 ins_pipe( pipe_slow );
7451 %}
7452
7453 // min = java.lang.Math.min(double a, double b)
7454 // max = java.lang.Math.max(double a, double b)
7455 instruct minmaxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp)
7456 %{
7457 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
7458 match(Set dst (MaxD a b));
7459 match(Set dst (MinD a b));
7460 effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
7461
7462 format %{ "minmaxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
7463 ins_encode %{
7464 int opcode = this->ideal_Opcode();
7465 int param_opcode = (opcode == Op_MinD) ? Op_MinV : Op_MaxV;
7466 __ vminmax_fp(param_opcode, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister,
7467 $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
7468 %}
7469 ins_pipe( pipe_slow );
7470 %}
7471
7472 instruct minmaxD_reduction_reg(legRegD dst, legRegD a, legRegD b, rRegL rtmp, rFlagsReg cr)
7473 %{
7474 predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
7475 match(Set dst (MaxD a b));
7476 match(Set dst (MinD a b));
7477 effect(USE a, USE b, TEMP rtmp, KILL cr);
7478
7479 format %{ "minmaxD_reduction $dst, $a, $b \t! using $rtmp as TEMP" %}
7480 ins_encode %{
7481 int opcode = this->ideal_Opcode();
7482 bool min = (opcode == Op_MinD) ? true : false;
7483 emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $rtmp$$Register,
7484 min, fp_prec_dbl /*pt*/);
7485 %}
7486 ins_pipe( pipe_slow );
7487 %}
7488
7489 // Load Effective Address
7490 instruct leaP8(rRegP dst, indOffset8 mem)
7491 %{
7492 match(Set dst mem);
7493
7494 ins_cost(110); // XXX
7495 format %{ "leaq $dst, $mem\t# ptr 8" %}
7496 ins_encode %{
7497 __ leaq($dst$$Register, $mem$$Address);
7498 %}
7499 ins_pipe(ialu_reg_reg_fat);
7500 %}
7501
7502 instruct leaP32(rRegP dst, indOffset32 mem)
7503 %{
7504 match(Set dst mem);
7505
7506 ins_cost(110);
7507 format %{ "leaq $dst, $mem\t# ptr 32" %}
7508 ins_encode %{
7509 __ leaq($dst$$Register, $mem$$Address);
7510 %}
7511 ins_pipe(ialu_reg_reg_fat);
7512 %}
7513
7514 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
7515 %{
7516 match(Set dst mem);
7517
7518 ins_cost(110);
7519 format %{ "leaq $dst, $mem\t# ptr idxoff" %}
7520 ins_encode %{
7521 __ leaq($dst$$Register, $mem$$Address);
7522 %}
7523 ins_pipe(ialu_reg_reg_fat);
7524 %}
7525
7526 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
7527 %{
7528 match(Set dst mem);
7529
7530 ins_cost(110);
7531 format %{ "leaq $dst, $mem\t# ptr idxscale" %}
7532 ins_encode %{
7533 __ leaq($dst$$Register, $mem$$Address);
7534 %}
7535 ins_pipe(ialu_reg_reg_fat);
7536 %}
7537
7538 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
7539 %{
7540 match(Set dst mem);
7541
7542 ins_cost(110);
7543 format %{ "leaq $dst, $mem\t# ptr idxscale" %}
7544 ins_encode %{
7545 __ leaq($dst$$Register, $mem$$Address);
7546 %}
7547 ins_pipe(ialu_reg_reg_fat);
7548 %}
7549
7550 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
7551 %{
7552 match(Set dst mem);
7553
7554 ins_cost(110);
7555 format %{ "leaq $dst, $mem\t# ptr idxscaleoff" %}
7556 ins_encode %{
7557 __ leaq($dst$$Register, $mem$$Address);
7558 %}
7559 ins_pipe(ialu_reg_reg_fat);
7560 %}
7561
7562 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
7563 %{
7564 match(Set dst mem);
7565
7566 ins_cost(110);
7567 format %{ "leaq $dst, $mem\t# ptr posidxoff" %}
7568 ins_encode %{
7569 __ leaq($dst$$Register, $mem$$Address);
7570 %}
7571 ins_pipe(ialu_reg_reg_fat);
7572 %}
7573
7574 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
7575 %{
7576 match(Set dst mem);
7577
7578 ins_cost(110);
7579 format %{ "leaq $dst, $mem\t# ptr posidxscaleoff" %}
7580 ins_encode %{
7581 __ leaq($dst$$Register, $mem$$Address);
7582 %}
7583 ins_pipe(ialu_reg_reg_fat);
7584 %}
7585
7586 // Load Effective Address which uses Narrow (32-bits) oop
7587 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
7588 %{
7589 predicate(UseCompressedOops && (CompressedOops::shift() != 0));
7590 match(Set dst mem);
7591
7592 ins_cost(110);
7593 format %{ "leaq $dst, $mem\t# ptr compressedoopoff32" %}
7594 ins_encode %{
7595 __ leaq($dst$$Register, $mem$$Address);
7596 %}
7597 ins_pipe(ialu_reg_reg_fat);
7598 %}
7599
7600 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
7601 %{
7602 predicate(CompressedOops::shift() == 0);
7603 match(Set dst mem);
7604
7605 ins_cost(110); // XXX
7606 format %{ "leaq $dst, $mem\t# ptr off8narrow" %}
7607 ins_encode %{
7608 __ leaq($dst$$Register, $mem$$Address);
7609 %}
7610 ins_pipe(ialu_reg_reg_fat);
7611 %}
7612
7613 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
7614 %{
7615 predicate(CompressedOops::shift() == 0);
7616 match(Set dst mem);
7617
7618 ins_cost(110);
7619 format %{ "leaq $dst, $mem\t# ptr off32narrow" %}
7620 ins_encode %{
7621 __ leaq($dst$$Register, $mem$$Address);
7622 %}
7623 ins_pipe(ialu_reg_reg_fat);
7624 %}
7625
7626 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
7627 %{
7628 predicate(CompressedOops::shift() == 0);
7629 match(Set dst mem);
7630
7631 ins_cost(110);
7632 format %{ "leaq $dst, $mem\t# ptr idxoffnarrow" %}
7633 ins_encode %{
7634 __ leaq($dst$$Register, $mem$$Address);
7635 %}
7636 ins_pipe(ialu_reg_reg_fat);
7637 %}
7638
7639 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
7640 %{
7641 predicate(CompressedOops::shift() == 0);
7642 match(Set dst mem);
7643
7644 ins_cost(110);
7645 format %{ "leaq $dst, $mem\t# ptr idxscalenarrow" %}
7646 ins_encode %{
7647 __ leaq($dst$$Register, $mem$$Address);
7648 %}
7649 ins_pipe(ialu_reg_reg_fat);
7650 %}
7651
7652 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
7653 %{
7654 predicate(CompressedOops::shift() == 0);
7655 match(Set dst mem);
7656
7657 ins_cost(110);
7658 format %{ "leaq $dst, $mem\t# ptr idxscaleoffnarrow" %}
7659 ins_encode %{
7660 __ leaq($dst$$Register, $mem$$Address);
7661 %}
7662 ins_pipe(ialu_reg_reg_fat);
7663 %}
7664
7665 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
7666 %{
7667 predicate(CompressedOops::shift() == 0);
7668 match(Set dst mem);
7669
7670 ins_cost(110);
7671 format %{ "leaq $dst, $mem\t# ptr posidxoffnarrow" %}
7672 ins_encode %{
7673 __ leaq($dst$$Register, $mem$$Address);
7674 %}
7675 ins_pipe(ialu_reg_reg_fat);
7676 %}
7677
7678 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
7679 %{
7680 predicate(CompressedOops::shift() == 0);
7681 match(Set dst mem);
7682
7683 ins_cost(110);
7684 format %{ "leaq $dst, $mem\t# ptr posidxscaleoffnarrow" %}
7685 ins_encode %{
7686 __ leaq($dst$$Register, $mem$$Address);
7687 %}
7688 ins_pipe(ialu_reg_reg_fat);
7689 %}
7690
7691 instruct loadConI(rRegI dst, immI src)
7692 %{
7693 match(Set dst src);
7694
7695 format %{ "movl $dst, $src\t# int" %}
7696 ins_encode %{
7697 __ movl($dst$$Register, $src$$constant);
7698 %}
7699 ins_pipe(ialu_reg_fat); // XXX
7700 %}
7701
7702 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
7703 %{
7704 match(Set dst src);
7705 effect(KILL cr);
7706
7707 ins_cost(50);
7708 format %{ "xorl $dst, $dst\t# int" %}
7709 ins_encode %{
7710 __ xorl($dst$$Register, $dst$$Register);
7711 %}
7712 ins_pipe(ialu_reg);
7713 %}
7714
7715 instruct loadConL(rRegL dst, immL src)
7716 %{
7717 match(Set dst src);
7718
7719 ins_cost(150);
7720 format %{ "movq $dst, $src\t# long" %}
7721 ins_encode %{
7722 __ mov64($dst$$Register, $src$$constant);
7723 %}
7724 ins_pipe(ialu_reg);
7725 %}
7726
7727 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
7728 %{
7729 match(Set dst src);
7730 effect(KILL cr);
7731
7732 ins_cost(50);
7733 format %{ "xorl $dst, $dst\t# long" %}
7734 ins_encode %{
7735 __ xorl($dst$$Register, $dst$$Register);
7736 %}
7737 ins_pipe(ialu_reg); // XXX
7738 %}
7739
7740 instruct loadConUL32(rRegL dst, immUL32 src)
7741 %{
7742 match(Set dst src);
7743
7744 ins_cost(60);
7745 format %{ "movl $dst, $src\t# long (unsigned 32-bit)" %}
7746 ins_encode %{
7747 __ movl($dst$$Register, $src$$constant);
7748 %}
7749 ins_pipe(ialu_reg);
7750 %}
7751
7752 instruct loadConL32(rRegL dst, immL32 src)
7753 %{
7754 match(Set dst src);
7755
7756 ins_cost(70);
7757 format %{ "movq $dst, $src\t# long (32-bit)" %}
7758 ins_encode %{
7759 __ movq($dst$$Register, $src$$constant);
7760 %}
7761 ins_pipe(ialu_reg);
7762 %}
7763
7764 instruct loadConP(rRegP dst, immP con) %{
7765 match(Set dst con);
7766
7767 format %{ "movq $dst, $con\t# ptr" %}
7768 ins_encode %{
7769 __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
7770 %}
7771 ins_pipe(ialu_reg_fat); // XXX
7772 %}
7773
7774 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
7775 %{
7776 match(Set dst src);
7777 effect(KILL cr);
7778
7779 ins_cost(50);
7780 format %{ "xorl $dst, $dst\t# ptr" %}
7781 ins_encode %{
7782 __ xorl($dst$$Register, $dst$$Register);
7783 %}
7784 ins_pipe(ialu_reg);
7785 %}
7786
7787 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
7788 %{
7789 match(Set dst src);
7790 effect(KILL cr);
7791
7792 ins_cost(60);
7793 format %{ "movl $dst, $src\t# ptr (positive 32-bit)" %}
7794 ins_encode %{
7795 __ movl($dst$$Register, $src$$constant);
7796 %}
7797 ins_pipe(ialu_reg);
7798 %}
7799
7800 instruct loadConF(regF dst, immF con) %{
7801 match(Set dst con);
7802 ins_cost(125);
7803 format %{ "movss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
7804 ins_encode %{
7805 __ movflt($dst$$XMMRegister, $constantaddress($con));
7806 %}
7807 ins_pipe(pipe_slow);
7808 %}
7809
7810 instruct loadConH(regF dst, immH con) %{
7811 match(Set dst con);
7812 ins_cost(125);
7813 format %{ "movss $dst, [$constantaddress]\t# load from constant table: halffloat=$con" %}
7814 ins_encode %{
7815 __ movflt($dst$$XMMRegister, $constantaddress($con));
7816 %}
7817 ins_pipe(pipe_slow);
7818 %}
7819
7820 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
7821 match(Set dst src);
7822 effect(KILL cr);
7823 format %{ "xorq $dst, $src\t# compressed null pointer" %}
7824 ins_encode %{
7825 __ xorq($dst$$Register, $dst$$Register);
7826 %}
7827 ins_pipe(ialu_reg);
7828 %}
7829
7830 instruct loadConN(rRegN dst, immN src) %{
7831 match(Set dst src);
7832
7833 ins_cost(125);
7834 format %{ "movl $dst, $src\t# compressed ptr" %}
7835 ins_encode %{
7836 address con = (address)$src$$constant;
7837 if (con == nullptr) {
7838 ShouldNotReachHere();
7839 } else {
7840 __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
7841 }
7842 %}
7843 ins_pipe(ialu_reg_fat); // XXX
7844 %}
7845
7846 instruct loadConNKlass(rRegN dst, immNKlass src) %{
7847 match(Set dst src);
7848
7849 ins_cost(125);
7850 format %{ "movl $dst, $src\t# compressed klass ptr" %}
7851 ins_encode %{
7852 address con = (address)$src$$constant;
7853 if (con == nullptr) {
7854 ShouldNotReachHere();
7855 } else {
7856 __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
7857 }
7858 %}
7859 ins_pipe(ialu_reg_fat); // XXX
7860 %}
7861
7862 instruct loadConF0(regF dst, immF0 src)
7863 %{
7864 match(Set dst src);
7865 ins_cost(100);
7866
7867 format %{ "xorps $dst, $dst\t# float 0.0" %}
7868 ins_encode %{
7869 __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
7870 %}
7871 ins_pipe(pipe_slow);
7872 %}
7873
7874 // Use the same format since predicate() can not be used here.
7875 instruct loadConD(regD dst, immD con) %{
7876 match(Set dst con);
7877 ins_cost(125);
7878 format %{ "movsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
7879 ins_encode %{
7880 __ movdbl($dst$$XMMRegister, $constantaddress($con));
7881 %}
7882 ins_pipe(pipe_slow);
7883 %}
7884
7885 instruct loadConD0(regD dst, immD0 src)
7886 %{
7887 match(Set dst src);
7888 ins_cost(100);
7889
7890 format %{ "xorpd $dst, $dst\t# double 0.0" %}
7891 ins_encode %{
7892 __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
7893 %}
7894 ins_pipe(pipe_slow);
7895 %}
7896
7897 instruct loadSSI(rRegI dst, stackSlotI src)
7898 %{
7899 match(Set dst src);
7900
7901 ins_cost(125);
7902 format %{ "movl $dst, $src\t# int stk" %}
7903 ins_encode %{
7904 __ movl($dst$$Register, $src$$Address);
7905 %}
7906 ins_pipe(ialu_reg_mem);
7907 %}
7908
7909 instruct loadSSL(rRegL dst, stackSlotL src)
7910 %{
7911 match(Set dst src);
7912
7913 ins_cost(125);
7914 format %{ "movq $dst, $src\t# long stk" %}
7915 ins_encode %{
7916 __ movq($dst$$Register, $src$$Address);
7917 %}
7918 ins_pipe(ialu_reg_mem);
7919 %}
7920
7921 instruct loadSSP(rRegP dst, stackSlotP src)
7922 %{
7923 match(Set dst src);
7924
7925 ins_cost(125);
7926 format %{ "movq $dst, $src\t# ptr stk" %}
7927 ins_encode %{
7928 __ movq($dst$$Register, $src$$Address);
7929 %}
7930 ins_pipe(ialu_reg_mem);
7931 %}
7932
7933 instruct loadSSF(regF dst, stackSlotF src)
7934 %{
7935 match(Set dst src);
7936
7937 ins_cost(125);
7938 format %{ "movss $dst, $src\t# float stk" %}
7939 ins_encode %{
7940 __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
7941 %}
7942 ins_pipe(pipe_slow); // XXX
7943 %}
7944
7945 // Use the same format since predicate() can not be used here.
7946 instruct loadSSD(regD dst, stackSlotD src)
7947 %{
7948 match(Set dst src);
7949
7950 ins_cost(125);
7951 format %{ "movsd $dst, $src\t# double stk" %}
7952 ins_encode %{
7953 __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
7954 %}
7955 ins_pipe(pipe_slow); // XXX
7956 %}
7957
7958 // Prefetch instructions for allocation.
7959 // Must be safe to execute with invalid address (cannot fault).
7960
7961 instruct prefetchAlloc( memory mem ) %{
7962 predicate(AllocatePrefetchInstr==3);
7963 match(PrefetchAllocation mem);
7964 ins_cost(125);
7965
7966 format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
7967 ins_encode %{
7968 __ prefetchw($mem$$Address);
7969 %}
7970 ins_pipe(ialu_mem);
7971 %}
7972
7973 instruct prefetchAllocNTA( memory mem ) %{
7974 predicate(AllocatePrefetchInstr==0);
7975 match(PrefetchAllocation mem);
7976 ins_cost(125);
7977
7978 format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
7979 ins_encode %{
7980 __ prefetchnta($mem$$Address);
7981 %}
7982 ins_pipe(ialu_mem);
7983 %}
7984
7985 instruct prefetchAllocT0( memory mem ) %{
7986 predicate(AllocatePrefetchInstr==1);
7987 match(PrefetchAllocation mem);
7988 ins_cost(125);
7989
7990 format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
7991 ins_encode %{
7992 __ prefetcht0($mem$$Address);
7993 %}
7994 ins_pipe(ialu_mem);
7995 %}
7996
7997 instruct prefetchAllocT2( memory mem ) %{
7998 predicate(AllocatePrefetchInstr==2);
7999 match(PrefetchAllocation mem);
8000 ins_cost(125);
8001
8002 format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
8003 ins_encode %{
8004 __ prefetcht2($mem$$Address);
8005 %}
8006 ins_pipe(ialu_mem);
8007 %}
8008
8009 //----------Store Instructions-------------------------------------------------
8010
8011 // Store Byte
8012 instruct storeB(memory mem, rRegI src)
8013 %{
8014 match(Set mem (StoreB mem src));
8015
8016 ins_cost(125); // XXX
8017 format %{ "movb $mem, $src\t# byte" %}
8018 ins_encode %{
8019 __ movb($mem$$Address, $src$$Register);
8020 %}
8021 ins_pipe(ialu_mem_reg);
8022 %}
8023
8024 // Store Char/Short
8025 instruct storeC(memory mem, rRegI src)
8026 %{
8027 match(Set mem (StoreC mem src));
8028
8029 ins_cost(125); // XXX
8030 format %{ "movw $mem, $src\t# char/short" %}
8031 ins_encode %{
8032 __ movw($mem$$Address, $src$$Register);
8033 %}
8034 ins_pipe(ialu_mem_reg);
8035 %}
8036
8037 // Store Integer
8038 instruct storeI(memory mem, rRegI src)
8039 %{
8040 match(Set mem (StoreI mem src));
8041
8042 ins_cost(125); // XXX
8043 format %{ "movl $mem, $src\t# int" %}
8044 ins_encode %{
8045 __ movl($mem$$Address, $src$$Register);
8046 %}
8047 ins_pipe(ialu_mem_reg);
8048 %}
8049
8050 // Store Long
8051 instruct storeL(memory mem, rRegL src)
8052 %{
8053 match(Set mem (StoreL mem src));
8054
8055 ins_cost(125); // XXX
8056 format %{ "movq $mem, $src\t# long" %}
8057 ins_encode %{
8058 __ movq($mem$$Address, $src$$Register);
8059 %}
8060 ins_pipe(ialu_mem_reg); // XXX
8061 %}
8062
8063 // Store Pointer
8064 instruct storeP(memory mem, any_RegP src)
8065 %{
8066 predicate(n->as_Store()->barrier_data() == 0);
8067 match(Set mem (StoreP mem src));
8068
8069 ins_cost(125); // XXX
8070 format %{ "movq $mem, $src\t# ptr" %}
8071 ins_encode %{
8072 __ movq($mem$$Address, $src$$Register);
8073 %}
8074 ins_pipe(ialu_mem_reg);
8075 %}
8076
8077 instruct storeImmP0(memory mem, immP0 zero)
8078 %{
8079 predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
8080 match(Set mem (StoreP mem zero));
8081
8082 ins_cost(125); // XXX
8083 format %{ "movq $mem, R12\t# ptr (R12_heapbase==0)" %}
8084 ins_encode %{
8085 __ movq($mem$$Address, r12);
8086 %}
8087 ins_pipe(ialu_mem_reg);
8088 %}
8089
8090 // Store Null Pointer, mark word, or other simple pointer constant.
8091 instruct storeImmP(memory mem, immP31 src)
8092 %{
8093 predicate(n->as_Store()->barrier_data() == 0);
8094 match(Set mem (StoreP mem src));
8095
8096 ins_cost(150); // XXX
8097 format %{ "movq $mem, $src\t# ptr" %}
8098 ins_encode %{
8099 __ movq($mem$$Address, $src$$constant);
8100 %}
8101 ins_pipe(ialu_mem_imm);
8102 %}
8103
8104 // Store Compressed Pointer
8105 instruct storeN(memory mem, rRegN src)
8106 %{
8107 predicate(n->as_Store()->barrier_data() == 0);
8108 match(Set mem (StoreN mem src));
8109
8110 ins_cost(125); // XXX
8111 format %{ "movl $mem, $src\t# compressed ptr" %}
8112 ins_encode %{
8113 __ movl($mem$$Address, $src$$Register);
8114 %}
8115 ins_pipe(ialu_mem_reg);
8116 %}
8117
8118 instruct storeNKlass(memory mem, rRegN src)
8119 %{
8120 match(Set mem (StoreNKlass mem src));
8121
8122 ins_cost(125); // XXX
8123 format %{ "movl $mem, $src\t# compressed klass ptr" %}
8124 ins_encode %{
8125 __ movl($mem$$Address, $src$$Register);
8126 %}
8127 ins_pipe(ialu_mem_reg);
8128 %}
8129
8130 instruct storeImmN0(memory mem, immN0 zero)
8131 %{
8132 predicate(CompressedOops::base() == nullptr && n->as_Store()->barrier_data() == 0);
8133 match(Set mem (StoreN mem zero));
8134
8135 ins_cost(125); // XXX
8136 format %{ "movl $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
8137 ins_encode %{
8138 __ movl($mem$$Address, r12);
8139 %}
8140 ins_pipe(ialu_mem_reg);
8141 %}
8142
8143 instruct storeImmN(memory mem, immN src)
8144 %{
8145 predicate(n->as_Store()->barrier_data() == 0);
8146 match(Set mem (StoreN mem src));
8147
8148 ins_cost(150); // XXX
8149 format %{ "movl $mem, $src\t# compressed ptr" %}
8150 ins_encode %{
8151 address con = (address)$src$$constant;
8152 if (con == nullptr) {
8153 __ movl($mem$$Address, 0);
8154 } else {
8155 __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
8156 }
8157 %}
8158 ins_pipe(ialu_mem_imm);
8159 %}
8160
8161 instruct storeImmNKlass(memory mem, immNKlass src)
8162 %{
8163 match(Set mem (StoreNKlass mem src));
8164
8165 ins_cost(150); // XXX
8166 format %{ "movl $mem, $src\t# compressed klass ptr" %}
8167 ins_encode %{
8168 __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
8169 %}
8170 ins_pipe(ialu_mem_imm);
8171 %}
8172
8173 // Store Integer Immediate
8174 instruct storeImmI0(memory mem, immI_0 zero)
8175 %{
8176 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8177 match(Set mem (StoreI mem zero));
8178
8179 ins_cost(125); // XXX
8180 format %{ "movl $mem, R12\t# int (R12_heapbase==0)" %}
8181 ins_encode %{
8182 __ movl($mem$$Address, r12);
8183 %}
8184 ins_pipe(ialu_mem_reg);
8185 %}
8186
8187 instruct storeImmI(memory mem, immI src)
8188 %{
8189 match(Set mem (StoreI mem src));
8190
8191 ins_cost(150);
8192 format %{ "movl $mem, $src\t# int" %}
8193 ins_encode %{
8194 __ movl($mem$$Address, $src$$constant);
8195 %}
8196 ins_pipe(ialu_mem_imm);
8197 %}
8198
8199 // Store Long Immediate
8200 instruct storeImmL0(memory mem, immL0 zero)
8201 %{
8202 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8203 match(Set mem (StoreL mem zero));
8204
8205 ins_cost(125); // XXX
8206 format %{ "movq $mem, R12\t# long (R12_heapbase==0)" %}
8207 ins_encode %{
8208 __ movq($mem$$Address, r12);
8209 %}
8210 ins_pipe(ialu_mem_reg);
8211 %}
8212
8213 instruct storeImmL(memory mem, immL32 src)
8214 %{
8215 match(Set mem (StoreL mem src));
8216
8217 ins_cost(150);
8218 format %{ "movq $mem, $src\t# long" %}
8219 ins_encode %{
8220 __ movq($mem$$Address, $src$$constant);
8221 %}
8222 ins_pipe(ialu_mem_imm);
8223 %}
8224
8225 // Store Short/Char Immediate
8226 instruct storeImmC0(memory mem, immI_0 zero)
8227 %{
8228 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8229 match(Set mem (StoreC mem zero));
8230
8231 ins_cost(125); // XXX
8232 format %{ "movw $mem, R12\t# short/char (R12_heapbase==0)" %}
8233 ins_encode %{
8234 __ movw($mem$$Address, r12);
8235 %}
8236 ins_pipe(ialu_mem_reg);
8237 %}
8238
8239 instruct storeImmI16(memory mem, immI16 src)
8240 %{
8241 predicate(UseStoreImmI16);
8242 match(Set mem (StoreC mem src));
8243
8244 ins_cost(150);
8245 format %{ "movw $mem, $src\t# short/char" %}
8246 ins_encode %{
8247 __ movw($mem$$Address, $src$$constant);
8248 %}
8249 ins_pipe(ialu_mem_imm);
8250 %}
8251
8252 // Store Byte Immediate
8253 instruct storeImmB0(memory mem, immI_0 zero)
8254 %{
8255 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8256 match(Set mem (StoreB mem zero));
8257
8258 ins_cost(125); // XXX
8259 format %{ "movb $mem, R12\t# short/char (R12_heapbase==0)" %}
8260 ins_encode %{
8261 __ movb($mem$$Address, r12);
8262 %}
8263 ins_pipe(ialu_mem_reg);
8264 %}
8265
8266 instruct storeImmB(memory mem, immI8 src)
8267 %{
8268 match(Set mem (StoreB mem src));
8269
8270 ins_cost(150); // XXX
8271 format %{ "movb $mem, $src\t# byte" %}
8272 ins_encode %{
8273 __ movb($mem$$Address, $src$$constant);
8274 %}
8275 ins_pipe(ialu_mem_imm);
8276 %}
8277
8278 // Store Float
8279 instruct storeF(memory mem, regF src)
8280 %{
8281 match(Set mem (StoreF mem src));
8282
8283 ins_cost(95); // XXX
8284 format %{ "movss $mem, $src\t# float" %}
8285 ins_encode %{
8286 __ movflt($mem$$Address, $src$$XMMRegister);
8287 %}
8288 ins_pipe(pipe_slow); // XXX
8289 %}
8290
8291 // Store immediate Float value (it is faster than store from XMM register)
8292 instruct storeF0(memory mem, immF0 zero)
8293 %{
8294 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8295 match(Set mem (StoreF mem zero));
8296
8297 ins_cost(25); // XXX
8298 format %{ "movl $mem, R12\t# float 0. (R12_heapbase==0)" %}
8299 ins_encode %{
8300 __ movl($mem$$Address, r12);
8301 %}
8302 ins_pipe(ialu_mem_reg);
8303 %}
8304
8305 instruct storeF_imm(memory mem, immF src)
8306 %{
8307 match(Set mem (StoreF mem src));
8308
8309 ins_cost(50);
8310 format %{ "movl $mem, $src\t# float" %}
8311 ins_encode %{
8312 __ movl($mem$$Address, jint_cast($src$$constant));
8313 %}
8314 ins_pipe(ialu_mem_imm);
8315 %}
8316
8317 // Store Double
8318 instruct storeD(memory mem, regD src)
8319 %{
8320 match(Set mem (StoreD mem src));
8321
8322 ins_cost(95); // XXX
8323 format %{ "movsd $mem, $src\t# double" %}
8324 ins_encode %{
8325 __ movdbl($mem$$Address, $src$$XMMRegister);
8326 %}
8327 ins_pipe(pipe_slow); // XXX
8328 %}
8329
8330 // Store immediate double 0.0 (it is faster than store from XMM register)
8331 instruct storeD0_imm(memory mem, immD0 src)
8332 %{
8333 predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
8334 match(Set mem (StoreD mem src));
8335
8336 ins_cost(50);
8337 format %{ "movq $mem, $src\t# double 0." %}
8338 ins_encode %{
8339 __ movq($mem$$Address, $src$$constant);
8340 %}
8341 ins_pipe(ialu_mem_imm);
8342 %}
8343
8344 instruct storeD0(memory mem, immD0 zero)
8345 %{
8346 predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
8347 match(Set mem (StoreD mem zero));
8348
8349 ins_cost(25); // XXX
8350 format %{ "movq $mem, R12\t# double 0. (R12_heapbase==0)" %}
8351 ins_encode %{
8352 __ movq($mem$$Address, r12);
8353 %}
8354 ins_pipe(ialu_mem_reg);
8355 %}
8356
8357 instruct storeSSI(stackSlotI dst, rRegI src)
8358 %{
8359 match(Set dst src);
8360
8361 ins_cost(100);
8362 format %{ "movl $dst, $src\t# int stk" %}
8363 ins_encode %{
8364 __ movl($dst$$Address, $src$$Register);
8365 %}
8366 ins_pipe( ialu_mem_reg );
8367 %}
8368
8369 instruct storeSSL(stackSlotL dst, rRegL src)
8370 %{
8371 match(Set dst src);
8372
8373 ins_cost(100);
8374 format %{ "movq $dst, $src\t# long stk" %}
8375 ins_encode %{
8376 __ movq($dst$$Address, $src$$Register);
8377 %}
8378 ins_pipe(ialu_mem_reg);
8379 %}
8380
8381 instruct storeSSP(stackSlotP dst, rRegP src)
8382 %{
8383 match(Set dst src);
8384
8385 ins_cost(100);
8386 format %{ "movq $dst, $src\t# ptr stk" %}
8387 ins_encode %{
8388 __ movq($dst$$Address, $src$$Register);
8389 %}
8390 ins_pipe(ialu_mem_reg);
8391 %}
8392
8393 instruct storeSSF(stackSlotF dst, regF src)
8394 %{
8395 match(Set dst src);
8396
8397 ins_cost(95); // XXX
8398 format %{ "movss $dst, $src\t# float stk" %}
8399 ins_encode %{
8400 __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
8401 %}
8402 ins_pipe(pipe_slow); // XXX
8403 %}
8404
8405 instruct storeSSD(stackSlotD dst, regD src)
8406 %{
8407 match(Set dst src);
8408
8409 ins_cost(95); // XXX
8410 format %{ "movsd $dst, $src\t# double stk" %}
8411 ins_encode %{
8412 __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
8413 %}
8414 ins_pipe(pipe_slow); // XXX
8415 %}
8416
8417 instruct cacheWB(indirect addr)
8418 %{
8419 predicate(VM_Version::supports_data_cache_line_flush());
8420 match(CacheWB addr);
8421
8422 ins_cost(100);
8423 format %{"cache wb $addr" %}
8424 ins_encode %{
8425 assert($addr->index_position() < 0, "should be");
8426 assert($addr$$disp == 0, "should be");
8427 __ cache_wb(Address($addr$$base$$Register, 0));
8428 %}
8429 ins_pipe(pipe_slow); // XXX
8430 %}
8431
8432 instruct cacheWBPreSync()
8433 %{
8434 predicate(VM_Version::supports_data_cache_line_flush());
8435 match(CacheWBPreSync);
8436
8437 ins_cost(100);
8438 format %{"cache wb presync" %}
8439 ins_encode %{
8440 __ cache_wbsync(true);
8441 %}
8442 ins_pipe(pipe_slow); // XXX
8443 %}
8444
8445 instruct cacheWBPostSync()
8446 %{
8447 predicate(VM_Version::supports_data_cache_line_flush());
8448 match(CacheWBPostSync);
8449
8450 ins_cost(100);
8451 format %{"cache wb postsync" %}
8452 ins_encode %{
8453 __ cache_wbsync(false);
8454 %}
8455 ins_pipe(pipe_slow); // XXX
8456 %}
8457
8458 //----------BSWAP Instructions-------------------------------------------------
8459 instruct bytes_reverse_int(rRegI dst) %{
8460 match(Set dst (ReverseBytesI dst));
8461
8462 format %{ "bswapl $dst" %}
8463 ins_encode %{
8464 __ bswapl($dst$$Register);
8465 %}
8466 ins_pipe( ialu_reg );
8467 %}
8468
8469 instruct bytes_reverse_long(rRegL dst) %{
8470 match(Set dst (ReverseBytesL dst));
8471
8472 format %{ "bswapq $dst" %}
8473 ins_encode %{
8474 __ bswapq($dst$$Register);
8475 %}
8476 ins_pipe( ialu_reg);
8477 %}
8478
8479 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
8480 match(Set dst (ReverseBytesUS dst));
8481 effect(KILL cr);
8482
8483 format %{ "bswapl $dst\n\t"
8484 "shrl $dst,16\n\t" %}
8485 ins_encode %{
8486 __ bswapl($dst$$Register);
8487 __ shrl($dst$$Register, 16);
8488 %}
8489 ins_pipe( ialu_reg );
8490 %}
8491
8492 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
8493 match(Set dst (ReverseBytesS dst));
8494 effect(KILL cr);
8495
8496 format %{ "bswapl $dst\n\t"
8497 "sar $dst,16\n\t" %}
8498 ins_encode %{
8499 __ bswapl($dst$$Register);
8500 __ sarl($dst$$Register, 16);
8501 %}
8502 ins_pipe( ialu_reg );
8503 %}
8504
8505 //---------- Zeros Count Instructions ------------------------------------------
8506
8507 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
8508 predicate(UseCountLeadingZerosInstruction);
8509 match(Set dst (CountLeadingZerosI src));
8510 effect(KILL cr);
8511
8512 format %{ "lzcntl $dst, $src\t# count leading zeros (int)" %}
8513 ins_encode %{
8514 __ lzcntl($dst$$Register, $src$$Register);
8515 %}
8516 ins_pipe(ialu_reg);
8517 %}
8518
8519 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
8520 predicate(UseCountLeadingZerosInstruction);
8521 match(Set dst (CountLeadingZerosI (LoadI src)));
8522 effect(KILL cr);
8523 ins_cost(175);
8524 format %{ "lzcntl $dst, $src\t# count leading zeros (int)" %}
8525 ins_encode %{
8526 __ lzcntl($dst$$Register, $src$$Address);
8527 %}
8528 ins_pipe(ialu_reg_mem);
8529 %}
8530
8531 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
8532 predicate(!UseCountLeadingZerosInstruction);
8533 match(Set dst (CountLeadingZerosI src));
8534 effect(KILL cr);
8535
8536 format %{ "bsrl $dst, $src\t# count leading zeros (int)\n\t"
8537 "jnz skip\n\t"
8538 "movl $dst, -1\n"
8539 "skip:\n\t"
8540 "negl $dst\n\t"
8541 "addl $dst, 31" %}
8542 ins_encode %{
8543 Register Rdst = $dst$$Register;
8544 Register Rsrc = $src$$Register;
8545 Label skip;
8546 __ bsrl(Rdst, Rsrc);
8547 __ jccb(Assembler::notZero, skip);
8548 __ movl(Rdst, -1);
8549 __ bind(skip);
8550 __ negl(Rdst);
8551 __ addl(Rdst, BitsPerInt - 1);
8552 %}
8553 ins_pipe(ialu_reg);
8554 %}
8555
8556 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
8557 predicate(UseCountLeadingZerosInstruction);
8558 match(Set dst (CountLeadingZerosL src));
8559 effect(KILL cr);
8560
8561 format %{ "lzcntq $dst, $src\t# count leading zeros (long)" %}
8562 ins_encode %{
8563 __ lzcntq($dst$$Register, $src$$Register);
8564 %}
8565 ins_pipe(ialu_reg);
8566 %}
8567
8568 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
8569 predicate(UseCountLeadingZerosInstruction);
8570 match(Set dst (CountLeadingZerosL (LoadL src)));
8571 effect(KILL cr);
8572 ins_cost(175);
8573 format %{ "lzcntq $dst, $src\t# count leading zeros (long)" %}
8574 ins_encode %{
8575 __ lzcntq($dst$$Register, $src$$Address);
8576 %}
8577 ins_pipe(ialu_reg_mem);
8578 %}
8579
8580 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
8581 predicate(!UseCountLeadingZerosInstruction);
8582 match(Set dst (CountLeadingZerosL src));
8583 effect(KILL cr);
8584
8585 format %{ "bsrq $dst, $src\t# count leading zeros (long)\n\t"
8586 "jnz skip\n\t"
8587 "movl $dst, -1\n"
8588 "skip:\n\t"
8589 "negl $dst\n\t"
8590 "addl $dst, 63" %}
8591 ins_encode %{
8592 Register Rdst = $dst$$Register;
8593 Register Rsrc = $src$$Register;
8594 Label skip;
8595 __ bsrq(Rdst, Rsrc);
8596 __ jccb(Assembler::notZero, skip);
8597 __ movl(Rdst, -1);
8598 __ bind(skip);
8599 __ negl(Rdst);
8600 __ addl(Rdst, BitsPerLong - 1);
8601 %}
8602 ins_pipe(ialu_reg);
8603 %}
8604
8605 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
8606 predicate(UseCountTrailingZerosInstruction);
8607 match(Set dst (CountTrailingZerosI src));
8608 effect(KILL cr);
8609
8610 format %{ "tzcntl $dst, $src\t# count trailing zeros (int)" %}
8611 ins_encode %{
8612 __ tzcntl($dst$$Register, $src$$Register);
8613 %}
8614 ins_pipe(ialu_reg);
8615 %}
8616
8617 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
8618 predicate(UseCountTrailingZerosInstruction);
8619 match(Set dst (CountTrailingZerosI (LoadI src)));
8620 effect(KILL cr);
8621 ins_cost(175);
8622 format %{ "tzcntl $dst, $src\t# count trailing zeros (int)" %}
8623 ins_encode %{
8624 __ tzcntl($dst$$Register, $src$$Address);
8625 %}
8626 ins_pipe(ialu_reg_mem);
8627 %}
8628
8629 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
8630 predicate(!UseCountTrailingZerosInstruction);
8631 match(Set dst (CountTrailingZerosI src));
8632 effect(KILL cr);
8633
8634 format %{ "bsfl $dst, $src\t# count trailing zeros (int)\n\t"
8635 "jnz done\n\t"
8636 "movl $dst, 32\n"
8637 "done:" %}
8638 ins_encode %{
8639 Register Rdst = $dst$$Register;
8640 Label done;
8641 __ bsfl(Rdst, $src$$Register);
8642 __ jccb(Assembler::notZero, done);
8643 __ movl(Rdst, BitsPerInt);
8644 __ bind(done);
8645 %}
8646 ins_pipe(ialu_reg);
8647 %}
8648
8649 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
8650 predicate(UseCountTrailingZerosInstruction);
8651 match(Set dst (CountTrailingZerosL src));
8652 effect(KILL cr);
8653
8654 format %{ "tzcntq $dst, $src\t# count trailing zeros (long)" %}
8655 ins_encode %{
8656 __ tzcntq($dst$$Register, $src$$Register);
8657 %}
8658 ins_pipe(ialu_reg);
8659 %}
8660
8661 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
8662 predicate(UseCountTrailingZerosInstruction);
8663 match(Set dst (CountTrailingZerosL (LoadL src)));
8664 effect(KILL cr);
8665 ins_cost(175);
8666 format %{ "tzcntq $dst, $src\t# count trailing zeros (long)" %}
8667 ins_encode %{
8668 __ tzcntq($dst$$Register, $src$$Address);
8669 %}
8670 ins_pipe(ialu_reg_mem);
8671 %}
8672
8673 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
8674 predicate(!UseCountTrailingZerosInstruction);
8675 match(Set dst (CountTrailingZerosL src));
8676 effect(KILL cr);
8677
8678 format %{ "bsfq $dst, $src\t# count trailing zeros (long)\n\t"
8679 "jnz done\n\t"
8680 "movl $dst, 64\n"
8681 "done:" %}
8682 ins_encode %{
8683 Register Rdst = $dst$$Register;
8684 Label done;
8685 __ bsfq(Rdst, $src$$Register);
8686 __ jccb(Assembler::notZero, done);
8687 __ movl(Rdst, BitsPerLong);
8688 __ bind(done);
8689 %}
8690 ins_pipe(ialu_reg);
8691 %}
8692
8693 //--------------- Reverse Operation Instructions ----------------
8694 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
8695 predicate(!VM_Version::supports_gfni());
8696 match(Set dst (ReverseI src));
8697 effect(TEMP dst, TEMP rtmp, KILL cr);
8698 format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
8699 ins_encode %{
8700 __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
8701 %}
8702 ins_pipe( ialu_reg );
8703 %}
8704
8705 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, vlRegF xtmp1, vlRegF xtmp2, rRegL rtmp, rFlagsReg cr) %{
8706 predicate(VM_Version::supports_gfni());
8707 match(Set dst (ReverseI src));
8708 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
8709 format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
8710 ins_encode %{
8711 __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
8712 %}
8713 ins_pipe( ialu_reg );
8714 %}
8715
8716 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
8717 predicate(!VM_Version::supports_gfni());
8718 match(Set dst (ReverseL src));
8719 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
8720 format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
8721 ins_encode %{
8722 __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
8723 %}
8724 ins_pipe( ialu_reg );
8725 %}
8726
8727 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, vlRegD xtmp1, vlRegD xtmp2, rRegL rtmp, rFlagsReg cr) %{
8728 predicate(VM_Version::supports_gfni());
8729 match(Set dst (ReverseL src));
8730 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
8731 format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
8732 ins_encode %{
8733 __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
8734 %}
8735 ins_pipe( ialu_reg );
8736 %}
8737
8738 //---------- Population Count Instructions -------------------------------------
8739
8740 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
8741 predicate(UsePopCountInstruction);
8742 match(Set dst (PopCountI src));
8743 effect(KILL cr);
8744
8745 format %{ "popcnt $dst, $src" %}
8746 ins_encode %{
8747 __ popcntl($dst$$Register, $src$$Register);
8748 %}
8749 ins_pipe(ialu_reg);
8750 %}
8751
8752 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
8753 predicate(UsePopCountInstruction);
8754 match(Set dst (PopCountI (LoadI mem)));
8755 effect(KILL cr);
8756
8757 format %{ "popcnt $dst, $mem" %}
8758 ins_encode %{
8759 __ popcntl($dst$$Register, $mem$$Address);
8760 %}
8761 ins_pipe(ialu_reg);
8762 %}
8763
8764 // Note: Long.bitCount(long) returns an int.
8765 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
8766 predicate(UsePopCountInstruction);
8767 match(Set dst (PopCountL src));
8768 effect(KILL cr);
8769
8770 format %{ "popcnt $dst, $src" %}
8771 ins_encode %{
8772 __ popcntq($dst$$Register, $src$$Register);
8773 %}
8774 ins_pipe(ialu_reg);
8775 %}
8776
8777 // Note: Long.bitCount(long) returns an int.
8778 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
8779 predicate(UsePopCountInstruction);
8780 match(Set dst (PopCountL (LoadL mem)));
8781 effect(KILL cr);
8782
8783 format %{ "popcnt $dst, $mem" %}
8784 ins_encode %{
8785 __ popcntq($dst$$Register, $mem$$Address);
8786 %}
8787 ins_pipe(ialu_reg);
8788 %}
8789
8790
8791 //----------MemBar Instructions-----------------------------------------------
8792 // Memory barrier flavors
8793
8794 instruct membar_acquire()
8795 %{
8796 match(MemBarAcquire);
8797 match(LoadFence);
8798 ins_cost(0);
8799
8800 size(0);
8801 format %{ "MEMBAR-acquire ! (empty encoding)" %}
8802 ins_encode();
8803 ins_pipe(empty);
8804 %}
8805
8806 instruct membar_acquire_lock()
8807 %{
8808 match(MemBarAcquireLock);
8809 ins_cost(0);
8810
8811 size(0);
8812 format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
8813 ins_encode();
8814 ins_pipe(empty);
8815 %}
8816
8817 instruct membar_release()
8818 %{
8819 match(MemBarRelease);
8820 match(StoreFence);
8821 ins_cost(0);
8822
8823 size(0);
8824 format %{ "MEMBAR-release ! (empty encoding)" %}
8825 ins_encode();
8826 ins_pipe(empty);
8827 %}
8828
8829 instruct membar_release_lock()
8830 %{
8831 match(MemBarReleaseLock);
8832 ins_cost(0);
8833
8834 size(0);
8835 format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
8836 ins_encode();
8837 ins_pipe(empty);
8838 %}
8839
8840 instruct membar_storeload(rFlagsReg cr) %{
8841 match(MemBarStoreLoad);
8842 effect(KILL cr);
8843 ins_cost(400);
8844
8845 format %{
8846 $$template
8847 $$emit$$"lock addl [rsp + #0], 0\t! membar_storeload"
8848 %}
8849 ins_encode %{
8850 __ membar(Assembler::StoreLoad);
8851 %}
8852 ins_pipe(pipe_slow);
8853 %}
8854
8855 instruct membar_volatile(rFlagsReg cr) %{
8856 match(MemBarVolatile);
8857 effect(KILL cr);
8858 ins_cost(400);
8859
8860 format %{
8861 $$template
8862 $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
8863 %}
8864 ins_encode %{
8865 __ membar(Assembler::StoreLoad);
8866 %}
8867 ins_pipe(pipe_slow);
8868 %}
8869
8870 instruct unnecessary_membar_volatile()
8871 %{
8872 match(MemBarVolatile);
8873 predicate(Matcher::post_store_load_barrier(n));
8874 ins_cost(0);
8875
8876 size(0);
8877 format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
8878 ins_encode();
8879 ins_pipe(empty);
8880 %}
8881
8882 instruct membar_full(rFlagsReg cr) %{
8883 match(MemBarFull);
8884 effect(KILL cr);
8885 ins_cost(400);
8886
8887 format %{
8888 $$template
8889 $$emit$$"lock addl [rsp + #0], 0\t! membar_full"
8890 %}
8891 ins_encode %{
8892 __ membar(Assembler::StoreLoad);
8893 %}
8894 ins_pipe(pipe_slow);
8895 %}
8896
8897 instruct membar_storestore() %{
8898 match(MemBarStoreStore);
8899 match(StoreStoreFence);
8900 ins_cost(0);
8901
8902 size(0);
8903 format %{ "MEMBAR-storestore (empty encoding)" %}
8904 ins_encode( );
8905 ins_pipe(empty);
8906 %}
8907
8908 //----------Move Instructions--------------------------------------------------
8909
8910 instruct castX2P(rRegP dst, rRegL src)
8911 %{
8912 match(Set dst (CastX2P src));
8913
8914 format %{ "movq $dst, $src\t# long->ptr" %}
8915 ins_encode %{
8916 if ($dst$$reg != $src$$reg) {
8917 __ movptr($dst$$Register, $src$$Register);
8918 }
8919 %}
8920 ins_pipe(ialu_reg_reg); // XXX
8921 %}
8922
8923 instruct castP2X(rRegL dst, rRegP src)
8924 %{
8925 match(Set dst (CastP2X src));
8926
8927 format %{ "movq $dst, $src\t# ptr -> long" %}
8928 ins_encode %{
8929 if ($dst$$reg != $src$$reg) {
8930 __ movptr($dst$$Register, $src$$Register);
8931 }
8932 %}
8933 ins_pipe(ialu_reg_reg); // XXX
8934 %}
8935
8936 // Convert oop into int for vectors alignment masking
8937 instruct convP2I(rRegI dst, rRegP src)
8938 %{
8939 match(Set dst (ConvL2I (CastP2X src)));
8940
8941 format %{ "movl $dst, $src\t# ptr -> int" %}
8942 ins_encode %{
8943 __ movl($dst$$Register, $src$$Register);
8944 %}
8945 ins_pipe(ialu_reg_reg); // XXX
8946 %}
8947
8948 // Convert compressed oop into int for vectors alignment masking
8949 // in case of 32bit oops (heap < 4Gb).
8950 instruct convN2I(rRegI dst, rRegN src)
8951 %{
8952 predicate(CompressedOops::shift() == 0);
8953 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8954
8955 format %{ "movl $dst, $src\t# compressed ptr -> int" %}
8956 ins_encode %{
8957 __ movl($dst$$Register, $src$$Register);
8958 %}
8959 ins_pipe(ialu_reg_reg); // XXX
8960 %}
8961
8962 // Convert oop pointer into compressed form
8963 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
8964 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8965 match(Set dst (EncodeP src));
8966 effect(KILL cr);
8967 format %{ "encode_heap_oop $dst,$src" %}
8968 ins_encode %{
8969 Register s = $src$$Register;
8970 Register d = $dst$$Register;
8971 if (s != d) {
8972 __ movq(d, s);
8973 }
8974 __ encode_heap_oop(d);
8975 %}
8976 ins_pipe(ialu_reg_long);
8977 %}
8978
8979 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
8980 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8981 match(Set dst (EncodeP src));
8982 effect(KILL cr);
8983 format %{ "encode_heap_oop_not_null $dst,$src" %}
8984 ins_encode %{
8985 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8986 %}
8987 ins_pipe(ialu_reg_long);
8988 %}
8989
8990 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
8991 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8992 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8993 match(Set dst (DecodeN src));
8994 effect(KILL cr);
8995 format %{ "decode_heap_oop $dst,$src" %}
8996 ins_encode %{
8997 Register s = $src$$Register;
8998 Register d = $dst$$Register;
8999 if (s != d) {
9000 __ movq(d, s);
9001 }
9002 __ decode_heap_oop(d);
9003 %}
9004 ins_pipe(ialu_reg_long);
9005 %}
9006
9007 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
9008 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
9009 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
9010 match(Set dst (DecodeN src));
9011 effect(KILL cr);
9012 format %{ "decode_heap_oop_not_null $dst,$src" %}
9013 ins_encode %{
9014 Register s = $src$$Register;
9015 Register d = $dst$$Register;
9016 if (s != d) {
9017 __ decode_heap_oop_not_null(d, s);
9018 } else {
9019 __ decode_heap_oop_not_null(d);
9020 }
9021 %}
9022 ins_pipe(ialu_reg_long);
9023 %}
9024
9025 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
9026 match(Set dst (EncodePKlass src));
9027 effect(TEMP dst, KILL cr);
9028 format %{ "encode_and_move_klass_not_null $dst,$src" %}
9029 ins_encode %{
9030 __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
9031 %}
9032 ins_pipe(ialu_reg_long);
9033 %}
9034
9035 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
9036 match(Set dst (DecodeNKlass src));
9037 effect(TEMP dst, KILL cr);
9038 format %{ "decode_and_move_klass_not_null $dst,$src" %}
9039 ins_encode %{
9040 __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
9041 %}
9042 ins_pipe(ialu_reg_long);
9043 %}
9044
9045 //----------Conditional Move---------------------------------------------------
9046 // Jump
9047 // dummy instruction for generating temp registers
9048 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
9049 match(Jump (LShiftL switch_val shift));
9050 ins_cost(350);
9051 predicate(false);
9052 effect(TEMP dest);
9053
9054 format %{ "leaq $dest, [$constantaddress]\n\t"
9055 "jmp [$dest + $switch_val << $shift]\n\t" %}
9056 ins_encode %{
9057 // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
9058 // to do that and the compiler is using that register as one it can allocate.
9059 // So we build it all by hand.
9060 // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
9061 // ArrayAddress dispatch(table, index);
9062 Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
9063 __ lea($dest$$Register, $constantaddress);
9064 __ jmp(dispatch);
9065 %}
9066 ins_pipe(pipe_jmp);
9067 %}
9068
9069 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
9070 match(Jump (AddL (LShiftL switch_val shift) offset));
9071 ins_cost(350);
9072 effect(TEMP dest);
9073
9074 format %{ "leaq $dest, [$constantaddress]\n\t"
9075 "jmp [$dest + $switch_val << $shift + $offset]\n\t" %}
9076 ins_encode %{
9077 // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
9078 // to do that and the compiler is using that register as one it can allocate.
9079 // So we build it all by hand.
9080 // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
9081 // ArrayAddress dispatch(table, index);
9082 Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
9083 __ lea($dest$$Register, $constantaddress);
9084 __ jmp(dispatch);
9085 %}
9086 ins_pipe(pipe_jmp);
9087 %}
9088
9089 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
9090 match(Jump switch_val);
9091 ins_cost(350);
9092 effect(TEMP dest);
9093
9094 format %{ "leaq $dest, [$constantaddress]\n\t"
9095 "jmp [$dest + $switch_val]\n\t" %}
9096 ins_encode %{
9097 // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
9098 // to do that and the compiler is using that register as one it can allocate.
9099 // So we build it all by hand.
9100 // Address index(noreg, switch_reg, Address::times_1);
9101 // ArrayAddress dispatch(table, index);
9102 Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
9103 __ lea($dest$$Register, $constantaddress);
9104 __ jmp(dispatch);
9105 %}
9106 ins_pipe(pipe_jmp);
9107 %}
9108
9109 // Conditional move
9110 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
9111 %{
9112 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9113 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9114
9115 ins_cost(100); // XXX
9116 format %{ "setbn$cop $dst\t# signed, int" %}
9117 ins_encode %{
9118 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9119 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9120 %}
9121 ins_pipe(ialu_reg);
9122 %}
9123
9124 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
9125 %{
9126 predicate(!UseAPX);
9127 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9128
9129 ins_cost(200); // XXX
9130 format %{ "cmovl$cop $dst, $src\t# signed, int" %}
9131 ins_encode %{
9132 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9133 %}
9134 ins_pipe(pipe_cmov_reg);
9135 %}
9136
9137 instruct cmovI_reg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr, cmpOp cop)
9138 %{
9139 predicate(UseAPX);
9140 match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
9141
9142 ins_cost(200);
9143 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
9144 ins_encode %{
9145 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9146 %}
9147 ins_pipe(pipe_cmov_reg);
9148 %}
9149
9150 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
9151 %{
9152 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9153 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9154
9155 ins_cost(100); // XXX
9156 format %{ "setbn$cop $dst\t# unsigned, int" %}
9157 ins_encode %{
9158 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9159 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9160 %}
9161 ins_pipe(ialu_reg);
9162 %}
9163
9164 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
9165 predicate(!UseAPX);
9166 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9167
9168 ins_cost(200); // XXX
9169 format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
9170 ins_encode %{
9171 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9172 %}
9173 ins_pipe(pipe_cmov_reg);
9174 %}
9175
9176 instruct cmovI_regU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, rRegI src2) %{
9177 predicate(UseAPX);
9178 match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
9179
9180 ins_cost(200);
9181 format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
9182 ins_encode %{
9183 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9184 %}
9185 ins_pipe(pipe_cmov_reg);
9186 %}
9187
9188 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
9189 %{
9190 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9191 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9192
9193 ins_cost(100); // XXX
9194 format %{ "setbn$cop $dst\t# unsigned, int" %}
9195 ins_encode %{
9196 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9197 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9198 %}
9199 ins_pipe(ialu_reg);
9200 %}
9201
9202 instruct cmovI_imm_01UCFE(rRegI dst, immI_1 src, rFlagsRegUCFE cr, cmpOpUCFE cop)
9203 %{
9204 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
9205 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9206
9207 ins_cost(100); // XXX
9208 format %{ "setbn$cop $dst\t# signed, unsigned, int" %}
9209 ins_encode %{
9210 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9211 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9212 %}
9213 ins_pipe(ialu_reg);
9214 %}
9215
9216 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
9217 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9218
9219 ins_cost(200);
9220 expand %{
9221 cmovI_regU(cop, cr, dst, src);
9222 %}
9223 %}
9224
9225 instruct cmovI_regUCFE_ndd(rRegI dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegI src1, rRegI src2) %{
9226 match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
9227
9228 ins_cost(200);
9229 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, unsigned, int ndd" %}
9230 ins_encode %{
9231 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9232 %}
9233 ins_pipe(pipe_cmov_reg);
9234 %}
9235
9236 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
9237 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9238 match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
9239
9240 ins_cost(200); // XXX
9241 format %{ "cmovpl $dst, $src\n\t"
9242 "cmovnel $dst, $src" %}
9243 ins_encode %{
9244 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9245 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9246 %}
9247 ins_pipe(pipe_cmov_reg);
9248 %}
9249
9250 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9251 // inputs of the CMove
9252 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
9253 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9254 match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
9255 effect(TEMP dst);
9256
9257 ins_cost(200); // XXX
9258 format %{ "cmovpl $dst, $src\n\t"
9259 "cmovnel $dst, $src" %}
9260 ins_encode %{
9261 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9262 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9263 %}
9264 ins_pipe(pipe_cmov_reg);
9265 %}
9266
9267 // Conditional move
9268 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
9269 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9270
9271 ins_cost(250); // XXX
9272 format %{ "cmovl$cop $dst, $src\t# signed, int" %}
9273 ins_encode %{
9274 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9275 %}
9276 ins_pipe(pipe_cmov_mem);
9277 %}
9278
9279 // Conditional move
9280 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
9281 %{
9282 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9283
9284 ins_cost(250); // XXX
9285 format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
9286 ins_encode %{
9287 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9288 %}
9289 ins_pipe(pipe_cmov_mem);
9290 %}
9291
9292 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
9293 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9294
9295 ins_cost(250);
9296 expand %{
9297 cmovI_memU(cop, cr, dst, src);
9298 %}
9299 %}
9300
9301 instruct cmovI_memUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, rRegI dst, memory src) %{
9302 match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
9303
9304 ins_cost(250); // XXX
9305 format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
9306 ins_encode %{
9307 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9308 %}
9309 ins_pipe(pipe_cmov_mem);
9310 %}
9311
9312 // Conditional move
9313 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
9314 %{
9315 predicate(!UseAPX);
9316 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9317
9318 ins_cost(200); // XXX
9319 format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
9320 ins_encode %{
9321 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9322 %}
9323 ins_pipe(pipe_cmov_reg);
9324 %}
9325
9326 // Conditional move ndd
9327 instruct cmovN_reg_ndd(rRegN dst, rRegN src1, rRegN src2, rFlagsReg cr, cmpOp cop)
9328 %{
9329 predicate(UseAPX);
9330 match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
9331
9332 ins_cost(200);
9333 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, compressed ptr ndd" %}
9334 ins_encode %{
9335 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9336 %}
9337 ins_pipe(pipe_cmov_reg);
9338 %}
9339
9340 // Conditional move
9341 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
9342 %{
9343 predicate(!UseAPX);
9344 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9345
9346 ins_cost(200); // XXX
9347 format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
9348 ins_encode %{
9349 __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9350 %}
9351 ins_pipe(pipe_cmov_reg);
9352 %}
9353
9354 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
9355 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9356
9357 ins_cost(200);
9358 expand %{
9359 cmovN_regU(cop, cr, dst, src);
9360 %}
9361 %}
9362
9363 // Conditional move ndd
9364 instruct cmovN_regU_ndd(rRegN dst, cmpOpU cop, rFlagsRegU cr, rRegN src1, rRegN src2)
9365 %{
9366 predicate(UseAPX);
9367 match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
9368
9369 ins_cost(200);
9370 format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
9371 ins_encode %{
9372 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9373 %}
9374 ins_pipe(pipe_cmov_reg);
9375 %}
9376
9377 instruct cmovN_regUCFE_ndd(rRegN dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegN src1, rRegN src2) %{
9378 match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
9379
9380 ins_cost(200);
9381 format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, unsigned, compressed ptr ndd" %}
9382 ins_encode %{
9383 __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9384 %}
9385 ins_pipe(pipe_cmov_reg);
9386 %}
9387
9388 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
9389 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9390 match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
9391
9392 ins_cost(200); // XXX
9393 format %{ "cmovpl $dst, $src\n\t"
9394 "cmovnel $dst, $src" %}
9395 ins_encode %{
9396 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9397 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9398 %}
9399 ins_pipe(pipe_cmov_reg);
9400 %}
9401
9402 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9403 // inputs of the CMove
9404 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
9405 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9406 match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
9407
9408 ins_cost(200); // XXX
9409 format %{ "cmovpl $dst, $src\n\t"
9410 "cmovnel $dst, $src" %}
9411 ins_encode %{
9412 __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
9413 __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
9414 %}
9415 ins_pipe(pipe_cmov_reg);
9416 %}
9417
9418 // Conditional move
9419 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
9420 %{
9421 predicate(!UseAPX);
9422 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9423
9424 ins_cost(200); // XXX
9425 format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
9426 ins_encode %{
9427 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9428 %}
9429 ins_pipe(pipe_cmov_reg); // XXX
9430 %}
9431
9432 // Conditional move ndd
9433 instruct cmovP_reg_ndd(rRegP dst, rRegP src1, rRegP src2, rFlagsReg cr, cmpOp cop)
9434 %{
9435 predicate(UseAPX);
9436 match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
9437
9438 ins_cost(200);
9439 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, ptr ndd" %}
9440 ins_encode %{
9441 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9442 %}
9443 ins_pipe(pipe_cmov_reg);
9444 %}
9445
9446 // Conditional move
9447 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
9448 %{
9449 predicate(!UseAPX);
9450 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9451
9452 ins_cost(200); // XXX
9453 format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
9454 ins_encode %{
9455 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9456 %}
9457 ins_pipe(pipe_cmov_reg); // XXX
9458 %}
9459
9460 // Conditional move ndd
9461 instruct cmovP_regU_ndd(rRegP dst, cmpOpU cop, rFlagsRegU cr, rRegP src1, rRegP src2)
9462 %{
9463 predicate(UseAPX);
9464 match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
9465
9466 ins_cost(200);
9467 format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
9468 ins_encode %{
9469 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9470 %}
9471 ins_pipe(pipe_cmov_reg);
9472 %}
9473
9474 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
9475 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9476
9477 ins_cost(200);
9478 expand %{
9479 cmovP_regU(cop, cr, dst, src);
9480 %}
9481 %}
9482
9483 instruct cmovP_regUCFE_ndd(rRegP dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegP src1, rRegP src2) %{
9484 match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
9485
9486 ins_cost(200);
9487 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, unsigned, ptr ndd" %}
9488 ins_encode %{
9489 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9490 %}
9491 ins_pipe(pipe_cmov_reg);
9492 %}
9493
9494 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
9495 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9496 match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
9497
9498 ins_cost(200); // XXX
9499 format %{ "cmovpq $dst, $src\n\t"
9500 "cmovneq $dst, $src" %}
9501 ins_encode %{
9502 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9503 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9504 %}
9505 ins_pipe(pipe_cmov_reg);
9506 %}
9507
9508 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9509 // inputs of the CMove
9510 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
9511 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9512 match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
9513
9514 ins_cost(200); // XXX
9515 format %{ "cmovpq $dst, $src\n\t"
9516 "cmovneq $dst, $src" %}
9517 ins_encode %{
9518 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9519 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9520 %}
9521 ins_pipe(pipe_cmov_reg);
9522 %}
9523
9524 instruct cmovL_imm_01(rRegL dst, immL1 src, rFlagsReg cr, cmpOp cop)
9525 %{
9526 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9527 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9528
9529 ins_cost(100); // XXX
9530 format %{ "setbn$cop $dst\t# signed, long" %}
9531 ins_encode %{
9532 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9533 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9534 %}
9535 ins_pipe(ialu_reg);
9536 %}
9537
9538 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
9539 %{
9540 predicate(!UseAPX);
9541 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9542
9543 ins_cost(200); // XXX
9544 format %{ "cmovq$cop $dst, $src\t# signed, long" %}
9545 ins_encode %{
9546 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9547 %}
9548 ins_pipe(pipe_cmov_reg); // XXX
9549 %}
9550
9551 instruct cmovL_reg_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, rRegL src2)
9552 %{
9553 predicate(UseAPX);
9554 match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
9555
9556 ins_cost(200);
9557 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
9558 ins_encode %{
9559 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9560 %}
9561 ins_pipe(pipe_cmov_reg);
9562 %}
9563
9564 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
9565 %{
9566 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9567
9568 ins_cost(200); // XXX
9569 format %{ "cmovq$cop $dst, $src\t# signed, long" %}
9570 ins_encode %{
9571 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9572 %}
9573 ins_pipe(pipe_cmov_mem); // XXX
9574 %}
9575
9576 instruct cmovL_imm_01U(rRegL dst, immL1 src, rFlagsRegU cr, cmpOpU cop)
9577 %{
9578 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9579 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9580
9581 ins_cost(100); // XXX
9582 format %{ "setbn$cop $dst\t# unsigned, long" %}
9583 ins_encode %{
9584 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9585 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9586 %}
9587 ins_pipe(ialu_reg);
9588 %}
9589
9590 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
9591 %{
9592 predicate(!UseAPX);
9593 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9594
9595 ins_cost(200); // XXX
9596 format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
9597 ins_encode %{
9598 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
9599 %}
9600 ins_pipe(pipe_cmov_reg); // XXX
9601 %}
9602
9603 instruct cmovL_regU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, rRegL src2)
9604 %{
9605 predicate(UseAPX);
9606 match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
9607
9608 ins_cost(200);
9609 format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
9610 ins_encode %{
9611 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9612 %}
9613 ins_pipe(pipe_cmov_reg);
9614 %}
9615
9616 instruct cmovL_imm_01UCF(rRegL dst, immL1 src, rFlagsRegUCF cr, cmpOpUCF cop)
9617 %{
9618 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9619 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9620
9621 ins_cost(100); // XXX
9622 format %{ "setbn$cop $dst\t# unsigned, long" %}
9623 ins_encode %{
9624 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9625 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9626 %}
9627 ins_pipe(ialu_reg);
9628 %}
9629
9630 instruct cmovL_imm_01UCFE(rRegL dst, immL1 src, rFlagsRegUCFE cr, cmpOpUCFE cop)
9631 %{
9632 predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
9633 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9634
9635 ins_cost(100); // XXX
9636 format %{ "setbn$cop $dst\t# signed, unsigned, long" %}
9637 ins_encode %{
9638 Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
9639 __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
9640 %}
9641 ins_pipe(ialu_reg);
9642 %}
9643
9644 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
9645 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9646
9647 ins_cost(200);
9648 expand %{
9649 cmovL_regU(cop, cr, dst, src);
9650 %}
9651 %}
9652
9653 instruct cmovL_regUCFE_ndd(rRegL dst, cmpOpUCFE cop, rFlagsRegUCFE cr, rRegL src1, rRegL src2)
9654 %{
9655 match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
9656
9657 ins_cost(200);
9658 format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, unsigned, long ndd" %}
9659 ins_encode %{
9660 __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
9661 %}
9662 ins_pipe(pipe_cmov_reg);
9663 %}
9664
9665 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
9666 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
9667 match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
9668
9669 ins_cost(200); // XXX
9670 format %{ "cmovpq $dst, $src\n\t"
9671 "cmovneq $dst, $src" %}
9672 ins_encode %{
9673 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9674 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9675 %}
9676 ins_pipe(pipe_cmov_reg);
9677 %}
9678
9679 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
9680 // inputs of the CMove
9681 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
9682 predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
9683 match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
9684
9685 ins_cost(200); // XXX
9686 format %{ "cmovpq $dst, $src\n\t"
9687 "cmovneq $dst, $src" %}
9688 ins_encode %{
9689 __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
9690 __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
9691 %}
9692 ins_pipe(pipe_cmov_reg);
9693 %}
9694
9695 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
9696 %{
9697 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9698
9699 ins_cost(200); // XXX
9700 format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
9701 ins_encode %{
9702 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9703 %}
9704 ins_pipe(pipe_cmov_mem); // XXX
9705 %}
9706
9707 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
9708 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9709
9710 ins_cost(200);
9711 expand %{
9712 cmovL_memU(cop, cr, dst, src);
9713 %}
9714 %}
9715
9716 instruct cmovL_memUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, rRegL dst, memory src) %{
9717 match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
9718
9719 ins_cost(200); // XXX
9720 format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
9721 ins_encode %{
9722 __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
9723 %}
9724 ins_pipe(pipe_cmov_mem); // XXX
9725 %}
9726
9727 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
9728 %{
9729 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9730
9731 ins_cost(200); // XXX
9732 format %{ "jn$cop skip\t# signed cmove float\n\t"
9733 "movss $dst, $src\n"
9734 "skip:" %}
9735 ins_encode %{
9736 Label Lskip;
9737 // Invert sense of branch from sense of CMOV
9738 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9739 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
9740 __ bind(Lskip);
9741 %}
9742 ins_pipe(pipe_slow);
9743 %}
9744
9745 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
9746 %{
9747 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9748
9749 ins_cost(200); // XXX
9750 format %{ "jn$cop skip\t# unsigned cmove float\n\t"
9751 "movss $dst, $src\n"
9752 "skip:" %}
9753 ins_encode %{
9754 Label Lskip;
9755 // Invert sense of branch from sense of CMOV
9756 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9757 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
9758 __ bind(Lskip);
9759 %}
9760 ins_pipe(pipe_slow);
9761 %}
9762
9763 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
9764 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9765
9766 ins_cost(200);
9767 expand %{
9768 cmovF_regU(cop, cr, dst, src);
9769 %}
9770 %}
9771
9772 instruct cmovF_regUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, regF dst, regF src)
9773 %{
9774 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
9775
9776 ins_cost(200); // XXX
9777 format %{ "jn$cop skip\t# signed, unsigned cmove float\n\t"
9778 "movss $dst, $src\n"
9779 "skip:" %}
9780 ins_encode %{
9781 Label Lskip;
9782 // Invert sense of branch from sense of CMOV
9783 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9784 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
9785 __ bind(Lskip);
9786 %}
9787 ins_pipe(pipe_slow);
9788 %}
9789
9790 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
9791 %{
9792 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9793
9794 ins_cost(200); // XXX
9795 format %{ "jn$cop skip\t# signed cmove double\n\t"
9796 "movsd $dst, $src\n"
9797 "skip:" %}
9798 ins_encode %{
9799 Label Lskip;
9800 // Invert sense of branch from sense of CMOV
9801 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9802 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
9803 __ bind(Lskip);
9804 %}
9805 ins_pipe(pipe_slow);
9806 %}
9807
9808 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
9809 %{
9810 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9811
9812 ins_cost(200); // XXX
9813 format %{ "jn$cop skip\t# unsigned cmove double\n\t"
9814 "movsd $dst, $src\n"
9815 "skip:" %}
9816 ins_encode %{
9817 Label Lskip;
9818 // Invert sense of branch from sense of CMOV
9819 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9820 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
9821 __ bind(Lskip);
9822 %}
9823 ins_pipe(pipe_slow);
9824 %}
9825
9826 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
9827 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9828
9829 ins_cost(200);
9830 expand %{
9831 cmovD_regU(cop, cr, dst, src);
9832 %}
9833 %}
9834
9835 instruct cmovD_regUCFE(cmpOpUCFE cop, rFlagsRegUCFE cr, regD dst, regD src)
9836 %{
9837 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
9838
9839 ins_cost(200); // XXX
9840 format %{ "jn$cop skip\t# signed, unsigned cmove double\n\t"
9841 "movsd $dst, $src\n"
9842 "skip:" %}
9843 ins_encode %{
9844 Label Lskip;
9845 // Invert sense of branch from sense of CMOV
9846 __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
9847 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
9848 __ bind(Lskip);
9849 %}
9850 ins_pipe(pipe_slow);
9851 %}
9852
9853 //----------Arithmetic Instructions--------------------------------------------
9854 //----------Addition Instructions----------------------------------------------
9855
9856 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9857 %{
9858 predicate(!UseAPX);
9859 match(Set dst (AddI dst src));
9860 effect(KILL cr);
9861 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9862 format %{ "addl $dst, $src\t# int" %}
9863 ins_encode %{
9864 __ addl($dst$$Register, $src$$Register);
9865 %}
9866 ins_pipe(ialu_reg_reg);
9867 %}
9868
9869 instruct addI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
9870 %{
9871 predicate(UseAPX);
9872 match(Set dst (AddI src1 src2));
9873 effect(KILL cr);
9874 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
9875
9876 format %{ "eaddl $dst, $src1, $src2\t# int ndd" %}
9877 ins_encode %{
9878 __ eaddl($dst$$Register, $src1$$Register, $src2$$Register, false);
9879 %}
9880 ins_pipe(ialu_reg_reg);
9881 %}
9882
9883 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9884 %{
9885 predicate(!UseAPX);
9886 match(Set dst (AddI dst src));
9887 effect(KILL cr);
9888 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9889
9890 format %{ "addl $dst, $src\t# int" %}
9891 ins_encode %{
9892 __ addl($dst$$Register, $src$$constant);
9893 %}
9894 ins_pipe( ialu_reg );
9895 %}
9896
9897 instruct addI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
9898 %{
9899 predicate(UseAPX);
9900 match(Set dst (AddI src1 src2));
9901 effect(KILL cr);
9902 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
9903
9904 format %{ "eaddl $dst, $src1, $src2\t# int ndd" %}
9905 ins_encode %{
9906 __ eaddl($dst$$Register, $src1$$Register, $src2$$constant, false);
9907 %}
9908 ins_pipe( ialu_reg );
9909 %}
9910
9911 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9912 %{
9913 match(Set dst (AddI dst (LoadI src)));
9914 effect(KILL cr);
9915 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9916
9917 ins_cost(150); // XXX
9918 format %{ "addl $dst, $src\t# int" %}
9919 ins_encode %{
9920 __ addl($dst$$Register, $src$$Address);
9921 %}
9922 ins_pipe(ialu_reg_mem);
9923 %}
9924
9925 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9926 %{
9927 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
9928 effect(KILL cr);
9929 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9930
9931 ins_cost(150); // XXX
9932 format %{ "addl $dst, $src\t# int" %}
9933 ins_encode %{
9934 __ addl($dst$$Address, $src$$Register);
9935 %}
9936 ins_pipe(ialu_mem_reg);
9937 %}
9938
9939 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
9940 %{
9941 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
9942 effect(KILL cr);
9943 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
9944
9945
9946 ins_cost(125); // XXX
9947 format %{ "addl $dst, $src\t# int" %}
9948 ins_encode %{
9949 __ addl($dst$$Address, $src$$constant);
9950 %}
9951 ins_pipe(ialu_mem_imm);
9952 %}
9953
9954 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
9955 %{
9956 predicate(!UseAPX && UseIncDec);
9957 match(Set dst (AddI dst src));
9958 effect(KILL cr);
9959
9960 format %{ "incl $dst\t# int" %}
9961 ins_encode %{
9962 __ incrementl($dst$$Register);
9963 %}
9964 ins_pipe(ialu_reg);
9965 %}
9966
9967 instruct incI_rReg_ndd(rRegI dst, rRegI src, immI_1 val, rFlagsReg cr)
9968 %{
9969 predicate(UseAPX && UseIncDec);
9970 match(Set dst (AddI src val));
9971 effect(KILL cr);
9972 flag(PD::Flag_ndd_demotable_opr1);
9973
9974 format %{ "eincl $dst, $src\t# int ndd" %}
9975 ins_encode %{
9976 __ eincl($dst$$Register, $src$$Register, false);
9977 %}
9978 ins_pipe(ialu_reg);
9979 %}
9980
9981 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
9982 %{
9983 predicate(UseIncDec);
9984 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
9985 effect(KILL cr);
9986
9987 ins_cost(125); // XXX
9988 format %{ "incl $dst\t# int" %}
9989 ins_encode %{
9990 __ incrementl($dst$$Address);
9991 %}
9992 ins_pipe(ialu_mem_imm);
9993 %}
9994
9995 // XXX why does that use AddI
9996 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
9997 %{
9998 predicate(!UseAPX && UseIncDec);
9999 match(Set dst (AddI dst src));
10000 effect(KILL cr);
10001
10002 format %{ "decl $dst\t# int" %}
10003 ins_encode %{
10004 __ decrementl($dst$$Register);
10005 %}
10006 ins_pipe(ialu_reg);
10007 %}
10008
10009 instruct decI_rReg_ndd(rRegI dst, rRegI src, immI_M1 val, rFlagsReg cr)
10010 %{
10011 predicate(UseAPX && UseIncDec);
10012 match(Set dst (AddI src val));
10013 effect(KILL cr);
10014 flag(PD::Flag_ndd_demotable_opr1);
10015
10016 format %{ "edecl $dst, $src\t# int ndd" %}
10017 ins_encode %{
10018 __ edecl($dst$$Register, $src$$Register, false);
10019 %}
10020 ins_pipe(ialu_reg);
10021 %}
10022
10023 // XXX why does that use AddI
10024 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
10025 %{
10026 predicate(UseIncDec);
10027 match(Set dst (StoreI dst (AddI (LoadI dst) src)));
10028 effect(KILL cr);
10029
10030 ins_cost(125); // XXX
10031 format %{ "decl $dst\t# int" %}
10032 ins_encode %{
10033 __ decrementl($dst$$Address);
10034 %}
10035 ins_pipe(ialu_mem_imm);
10036 %}
10037
10038 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
10039 %{
10040 predicate(VM_Version::supports_fast_2op_lea());
10041 match(Set dst (AddI (LShiftI index scale) disp));
10042
10043 format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
10044 ins_encode %{
10045 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10046 __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
10047 %}
10048 ins_pipe(ialu_reg_reg);
10049 %}
10050
10051 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
10052 %{
10053 predicate(VM_Version::supports_fast_3op_lea());
10054 match(Set dst (AddI (AddI base index) disp));
10055
10056 format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
10057 ins_encode %{
10058 __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
10059 %}
10060 ins_pipe(ialu_reg_reg);
10061 %}
10062
10063 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
10064 %{
10065 predicate(VM_Version::supports_fast_2op_lea());
10066 match(Set dst (AddI base (LShiftI index scale)));
10067
10068 format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
10069 ins_encode %{
10070 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10071 __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
10072 %}
10073 ins_pipe(ialu_reg_reg);
10074 %}
10075
10076 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
10077 %{
10078 predicate(VM_Version::supports_fast_3op_lea());
10079 match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
10080
10081 format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
10082 ins_encode %{
10083 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10084 __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
10085 %}
10086 ins_pipe(ialu_reg_reg);
10087 %}
10088
10089 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10090 %{
10091 predicate(!UseAPX);
10092 match(Set dst (AddL dst src));
10093 effect(KILL cr);
10094 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10095
10096 format %{ "addq $dst, $src\t# long" %}
10097 ins_encode %{
10098 __ addq($dst$$Register, $src$$Register);
10099 %}
10100 ins_pipe(ialu_reg_reg);
10101 %}
10102
10103 instruct addL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
10104 %{
10105 predicate(UseAPX);
10106 match(Set dst (AddL src1 src2));
10107 effect(KILL cr);
10108 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
10109
10110 format %{ "eaddq $dst, $src1, $src2\t# long ndd" %}
10111 ins_encode %{
10112 __ eaddq($dst$$Register, $src1$$Register, $src2$$Register, false);
10113 %}
10114 ins_pipe(ialu_reg_reg);
10115 %}
10116
10117 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10118 %{
10119 predicate(!UseAPX);
10120 match(Set dst (AddL dst src));
10121 effect(KILL cr);
10122 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10123
10124 format %{ "addq $dst, $src\t# long" %}
10125 ins_encode %{
10126 __ addq($dst$$Register, $src$$constant);
10127 %}
10128 ins_pipe( ialu_reg );
10129 %}
10130
10131 instruct addL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
10132 %{
10133 predicate(UseAPX);
10134 match(Set dst (AddL src1 src2));
10135 effect(KILL cr);
10136 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
10137
10138 format %{ "eaddq $dst, $src1, $src2\t# long ndd" %}
10139 ins_encode %{
10140 __ eaddq($dst$$Register, $src1$$Register, $src2$$constant, false);
10141 %}
10142 ins_pipe( ialu_reg );
10143 %}
10144
10145 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10146 %{
10147 match(Set dst (AddL dst (LoadL src)));
10148 effect(KILL cr);
10149 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10150
10151 ins_cost(150); // XXX
10152 format %{ "addq $dst, $src\t# long" %}
10153 ins_encode %{
10154 __ addq($dst$$Register, $src$$Address);
10155 %}
10156 ins_pipe(ialu_reg_mem);
10157 %}
10158
10159 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10160 %{
10161 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10162 effect(KILL cr);
10163 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10164
10165 ins_cost(150); // XXX
10166 format %{ "addq $dst, $src\t# long" %}
10167 ins_encode %{
10168 __ addq($dst$$Address, $src$$Register);
10169 %}
10170 ins_pipe(ialu_mem_reg);
10171 %}
10172
10173 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10174 %{
10175 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10176 effect(KILL cr);
10177 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10178
10179 ins_cost(125); // XXX
10180 format %{ "addq $dst, $src\t# long" %}
10181 ins_encode %{
10182 __ addq($dst$$Address, $src$$constant);
10183 %}
10184 ins_pipe(ialu_mem_imm);
10185 %}
10186
10187 instruct incL_rReg(rRegL dst, immL1 src, rFlagsReg cr)
10188 %{
10189 predicate(!UseAPX && UseIncDec);
10190 match(Set dst (AddL dst src));
10191 effect(KILL cr);
10192
10193 format %{ "incq $dst\t# long" %}
10194 ins_encode %{
10195 __ incrementq($dst$$Register);
10196 %}
10197 ins_pipe(ialu_reg);
10198 %}
10199
10200 instruct incL_rReg_ndd(rRegL dst, rRegI src, immL1 val, rFlagsReg cr)
10201 %{
10202 predicate(UseAPX && UseIncDec);
10203 match(Set dst (AddL src val));
10204 effect(KILL cr);
10205 flag(PD::Flag_ndd_demotable_opr1);
10206
10207 format %{ "eincq $dst, $src\t# long ndd" %}
10208 ins_encode %{
10209 __ eincq($dst$$Register, $src$$Register, false);
10210 %}
10211 ins_pipe(ialu_reg);
10212 %}
10213
10214 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
10215 %{
10216 predicate(UseIncDec);
10217 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10218 effect(KILL cr);
10219
10220 ins_cost(125); // XXX
10221 format %{ "incq $dst\t# long" %}
10222 ins_encode %{
10223 __ incrementq($dst$$Address);
10224 %}
10225 ins_pipe(ialu_mem_imm);
10226 %}
10227
10228 // XXX why does that use AddL
10229 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
10230 %{
10231 predicate(!UseAPX && UseIncDec);
10232 match(Set dst (AddL dst src));
10233 effect(KILL cr);
10234
10235 format %{ "decq $dst\t# long" %}
10236 ins_encode %{
10237 __ decrementq($dst$$Register);
10238 %}
10239 ins_pipe(ialu_reg);
10240 %}
10241
10242 instruct decL_rReg_ndd(rRegL dst, rRegL src, immL_M1 val, rFlagsReg cr)
10243 %{
10244 predicate(UseAPX && UseIncDec);
10245 match(Set dst (AddL src val));
10246 effect(KILL cr);
10247 flag(PD::Flag_ndd_demotable_opr1);
10248
10249 format %{ "edecq $dst, $src\t# long ndd" %}
10250 ins_encode %{
10251 __ edecq($dst$$Register, $src$$Register, false);
10252 %}
10253 ins_pipe(ialu_reg);
10254 %}
10255
10256 // XXX why does that use AddL
10257 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
10258 %{
10259 predicate(UseIncDec);
10260 match(Set dst (StoreL dst (AddL (LoadL dst) src)));
10261 effect(KILL cr);
10262
10263 ins_cost(125); // XXX
10264 format %{ "decq $dst\t# long" %}
10265 ins_encode %{
10266 __ decrementq($dst$$Address);
10267 %}
10268 ins_pipe(ialu_mem_imm);
10269 %}
10270
10271 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
10272 %{
10273 predicate(VM_Version::supports_fast_2op_lea());
10274 match(Set dst (AddL (LShiftL index scale) disp));
10275
10276 format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
10277 ins_encode %{
10278 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10279 __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
10280 %}
10281 ins_pipe(ialu_reg_reg);
10282 %}
10283
10284 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
10285 %{
10286 predicate(VM_Version::supports_fast_3op_lea());
10287 match(Set dst (AddL (AddL base index) disp));
10288
10289 format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
10290 ins_encode %{
10291 __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
10292 %}
10293 ins_pipe(ialu_reg_reg);
10294 %}
10295
10296 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
10297 %{
10298 predicate(VM_Version::supports_fast_2op_lea());
10299 match(Set dst (AddL base (LShiftL index scale)));
10300
10301 format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
10302 ins_encode %{
10303 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10304 __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
10305 %}
10306 ins_pipe(ialu_reg_reg);
10307 %}
10308
10309 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
10310 %{
10311 predicate(VM_Version::supports_fast_3op_lea());
10312 match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
10313
10314 format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
10315 ins_encode %{
10316 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
10317 __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
10318 %}
10319 ins_pipe(ialu_reg_reg);
10320 %}
10321
10322 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
10323 %{
10324 match(Set dst (AddP dst src));
10325 effect(KILL cr);
10326 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10327
10328 format %{ "addq $dst, $src\t# ptr" %}
10329 ins_encode %{
10330 __ addq($dst$$Register, $src$$Register);
10331 %}
10332 ins_pipe(ialu_reg_reg);
10333 %}
10334
10335 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
10336 %{
10337 match(Set dst (AddP dst src));
10338 effect(KILL cr);
10339 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10340
10341 format %{ "addq $dst, $src\t# ptr" %}
10342 ins_encode %{
10343 __ addq($dst$$Register, $src$$constant);
10344 %}
10345 ins_pipe( ialu_reg );
10346 %}
10347
10348 // XXX addP mem ops ????
10349
10350 instruct checkCastPP(rRegP dst)
10351 %{
10352 match(Set dst (CheckCastPP dst));
10353
10354 size(0);
10355 format %{ "# checkcastPP of $dst" %}
10356 ins_encode(/* empty encoding */);
10357 ins_pipe(empty);
10358 %}
10359
10360 instruct castPP(rRegP dst)
10361 %{
10362 match(Set dst (CastPP dst));
10363
10364 size(0);
10365 format %{ "# castPP of $dst" %}
10366 ins_encode(/* empty encoding */);
10367 ins_pipe(empty);
10368 %}
10369
10370 instruct castII(rRegI dst)
10371 %{
10372 predicate(VerifyConstraintCasts == 0);
10373 match(Set dst (CastII dst));
10374
10375 size(0);
10376 format %{ "# castII of $dst" %}
10377 ins_encode(/* empty encoding */);
10378 ins_cost(0);
10379 ins_pipe(empty);
10380 %}
10381
10382 instruct castII_checked(rRegI dst, rFlagsReg cr)
10383 %{
10384 predicate(VerifyConstraintCasts > 0);
10385 match(Set dst (CastII dst));
10386
10387 effect(KILL cr);
10388 format %{ "# cast_checked_II $dst" %}
10389 ins_encode %{
10390 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register);
10391 %}
10392 ins_pipe(pipe_slow);
10393 %}
10394
10395 instruct castLL(rRegL dst)
10396 %{
10397 predicate(VerifyConstraintCasts == 0);
10398 match(Set dst (CastLL dst));
10399
10400 size(0);
10401 format %{ "# castLL of $dst" %}
10402 ins_encode(/* empty encoding */);
10403 ins_cost(0);
10404 ins_pipe(empty);
10405 %}
10406
10407 instruct castLL_checked_L32(rRegL dst, rFlagsReg cr)
10408 %{
10409 predicate(VerifyConstraintCasts > 0 && castLL_is_imm32(n));
10410 match(Set dst (CastLL dst));
10411
10412 effect(KILL cr);
10413 format %{ "# cast_checked_LL $dst" %}
10414 ins_encode %{
10415 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, noreg);
10416 %}
10417 ins_pipe(pipe_slow);
10418 %}
10419
10420 instruct castLL_checked(rRegL dst, rRegL tmp, rFlagsReg cr)
10421 %{
10422 predicate(VerifyConstraintCasts > 0 && !castLL_is_imm32(n));
10423 match(Set dst (CastLL dst));
10424
10425 effect(KILL cr, TEMP tmp);
10426 format %{ "# cast_checked_LL $dst\tusing $tmp as TEMP" %}
10427 ins_encode %{
10428 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, $tmp$$Register);
10429 %}
10430 ins_pipe(pipe_slow);
10431 %}
10432
10433 instruct castFF(regF dst)
10434 %{
10435 match(Set dst (CastFF dst));
10436
10437 size(0);
10438 format %{ "# castFF of $dst" %}
10439 ins_encode(/* empty encoding */);
10440 ins_cost(0);
10441 ins_pipe(empty);
10442 %}
10443
10444 instruct castHH(regF dst)
10445 %{
10446 match(Set dst (CastHH dst));
10447
10448 size(0);
10449 format %{ "# castHH of $dst" %}
10450 ins_encode(/* empty encoding */);
10451 ins_cost(0);
10452 ins_pipe(empty);
10453 %}
10454
10455 instruct castDD(regD dst)
10456 %{
10457 match(Set dst (CastDD dst));
10458
10459 size(0);
10460 format %{ "# castDD of $dst" %}
10461 ins_encode(/* empty encoding */);
10462 ins_cost(0);
10463 ins_pipe(empty);
10464 %}
10465
10466 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
10467 instruct compareAndSwapP(rRegI res,
10468 memory mem_ptr,
10469 rax_RegP oldval, rRegP newval,
10470 rFlagsReg cr)
10471 %{
10472 predicate(n->as_LoadStore()->barrier_data() == 0);
10473 match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
10474 match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
10475 effect(KILL cr, KILL oldval);
10476
10477 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10478 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10479 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10480 ins_encode %{
10481 __ lock();
10482 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10483 __ setcc(Assembler::equal, $res$$Register);
10484 %}
10485 ins_pipe( pipe_cmpxchg );
10486 %}
10487
10488 instruct compareAndSwapL(rRegI res,
10489 memory mem_ptr,
10490 rax_RegL oldval, rRegL newval,
10491 rFlagsReg cr)
10492 %{
10493 match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
10494 match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
10495 effect(KILL cr, KILL oldval);
10496
10497 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10498 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10499 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10500 ins_encode %{
10501 __ lock();
10502 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10503 __ setcc(Assembler::equal, $res$$Register);
10504 %}
10505 ins_pipe( pipe_cmpxchg );
10506 %}
10507
10508 instruct compareAndSwapI(rRegI res,
10509 memory mem_ptr,
10510 rax_RegI oldval, rRegI newval,
10511 rFlagsReg cr)
10512 %{
10513 match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
10514 match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
10515 effect(KILL cr, KILL oldval);
10516
10517 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10518 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10519 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10520 ins_encode %{
10521 __ lock();
10522 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10523 __ setcc(Assembler::equal, $res$$Register);
10524 %}
10525 ins_pipe( pipe_cmpxchg );
10526 %}
10527
10528 instruct compareAndSwapB(rRegI res,
10529 memory mem_ptr,
10530 rax_RegI oldval, rRegI newval,
10531 rFlagsReg cr)
10532 %{
10533 match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
10534 match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
10535 effect(KILL cr, KILL oldval);
10536
10537 format %{ "cmpxchgb $mem_ptr,$newval\t# "
10538 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10539 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10540 ins_encode %{
10541 __ lock();
10542 __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
10543 __ setcc(Assembler::equal, $res$$Register);
10544 %}
10545 ins_pipe( pipe_cmpxchg );
10546 %}
10547
10548 instruct compareAndSwapS(rRegI res,
10549 memory mem_ptr,
10550 rax_RegI oldval, rRegI newval,
10551 rFlagsReg cr)
10552 %{
10553 match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
10554 match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
10555 effect(KILL cr, KILL oldval);
10556
10557 format %{ "cmpxchgw $mem_ptr,$newval\t# "
10558 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10559 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10560 ins_encode %{
10561 __ lock();
10562 __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
10563 __ setcc(Assembler::equal, $res$$Register);
10564 %}
10565 ins_pipe( pipe_cmpxchg );
10566 %}
10567
10568 instruct compareAndSwapN(rRegI res,
10569 memory mem_ptr,
10570 rax_RegN oldval, rRegN newval,
10571 rFlagsReg cr) %{
10572 predicate(n->as_LoadStore()->barrier_data() == 0);
10573 match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
10574 match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
10575 effect(KILL cr, KILL oldval);
10576
10577 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10578 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
10579 "setcc $res \t# emits sete + movzbl or setzue for APX" %}
10580 ins_encode %{
10581 __ lock();
10582 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10583 __ setcc(Assembler::equal, $res$$Register);
10584 %}
10585 ins_pipe( pipe_cmpxchg );
10586 %}
10587
10588 instruct compareAndExchangeB(
10589 memory mem_ptr,
10590 rax_RegI oldval, rRegI newval,
10591 rFlagsReg cr)
10592 %{
10593 match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
10594 effect(KILL cr);
10595
10596 format %{ "cmpxchgb $mem_ptr,$newval\t# "
10597 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10598 ins_encode %{
10599 __ lock();
10600 __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
10601 %}
10602 ins_pipe( pipe_cmpxchg );
10603 %}
10604
10605 instruct compareAndExchangeS(
10606 memory mem_ptr,
10607 rax_RegI oldval, rRegI newval,
10608 rFlagsReg cr)
10609 %{
10610 match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
10611 effect(KILL cr);
10612
10613 format %{ "cmpxchgw $mem_ptr,$newval\t# "
10614 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10615 ins_encode %{
10616 __ lock();
10617 __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
10618 %}
10619 ins_pipe( pipe_cmpxchg );
10620 %}
10621
10622 instruct compareAndExchangeI(
10623 memory mem_ptr,
10624 rax_RegI oldval, rRegI newval,
10625 rFlagsReg cr)
10626 %{
10627 match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
10628 effect(KILL cr);
10629
10630 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10631 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10632 ins_encode %{
10633 __ lock();
10634 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10635 %}
10636 ins_pipe( pipe_cmpxchg );
10637 %}
10638
10639 instruct compareAndExchangeL(
10640 memory mem_ptr,
10641 rax_RegL oldval, rRegL newval,
10642 rFlagsReg cr)
10643 %{
10644 match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
10645 effect(KILL cr);
10646
10647 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10648 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10649 ins_encode %{
10650 __ lock();
10651 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10652 %}
10653 ins_pipe( pipe_cmpxchg );
10654 %}
10655
10656 instruct compareAndExchangeN(
10657 memory mem_ptr,
10658 rax_RegN oldval, rRegN newval,
10659 rFlagsReg cr) %{
10660 predicate(n->as_LoadStore()->barrier_data() == 0);
10661 match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
10662 effect(KILL cr);
10663
10664 format %{ "cmpxchgl $mem_ptr,$newval\t# "
10665 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10666 ins_encode %{
10667 __ lock();
10668 __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
10669 %}
10670 ins_pipe( pipe_cmpxchg );
10671 %}
10672
10673 instruct compareAndExchangeP(
10674 memory mem_ptr,
10675 rax_RegP oldval, rRegP newval,
10676 rFlagsReg cr)
10677 %{
10678 predicate(n->as_LoadStore()->barrier_data() == 0);
10679 match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
10680 effect(KILL cr);
10681
10682 format %{ "cmpxchgq $mem_ptr,$newval\t# "
10683 "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
10684 ins_encode %{
10685 __ lock();
10686 __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
10687 %}
10688 ins_pipe( pipe_cmpxchg );
10689 %}
10690
10691 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
10692 predicate(n->as_LoadStore()->result_not_used());
10693 match(Set dummy (GetAndAddB mem add));
10694 effect(KILL cr);
10695 format %{ "addb_lock $mem, $add" %}
10696 ins_encode %{
10697 __ lock();
10698 __ addb($mem$$Address, $add$$Register);
10699 %}
10700 ins_pipe(pipe_cmpxchg);
10701 %}
10702
10703 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
10704 predicate(n->as_LoadStore()->result_not_used());
10705 match(Set dummy (GetAndAddB mem add));
10706 effect(KILL cr);
10707 format %{ "addb_lock $mem, $add" %}
10708 ins_encode %{
10709 __ lock();
10710 __ addb($mem$$Address, $add$$constant);
10711 %}
10712 ins_pipe(pipe_cmpxchg);
10713 %}
10714
10715 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
10716 predicate(!n->as_LoadStore()->result_not_used());
10717 match(Set newval (GetAndAddB mem newval));
10718 effect(KILL cr);
10719 format %{ "xaddb_lock $mem, $newval\t# $newval -> byte" %}
10720 ins_encode %{
10721 __ lock();
10722 __ xaddb($mem$$Address, $newval$$Register);
10723 __ narrow_subword_type($newval$$Register, T_BYTE);
10724 %}
10725 ins_pipe(pipe_cmpxchg);
10726 %}
10727
10728 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
10729 predicate(n->as_LoadStore()->result_not_used());
10730 match(Set dummy (GetAndAddS mem add));
10731 effect(KILL cr);
10732 format %{ "addw_lock $mem, $add" %}
10733 ins_encode %{
10734 __ lock();
10735 __ addw($mem$$Address, $add$$Register);
10736 %}
10737 ins_pipe(pipe_cmpxchg);
10738 %}
10739
10740 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
10741 predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
10742 match(Set dummy (GetAndAddS mem add));
10743 effect(KILL cr);
10744 format %{ "addw_lock $mem, $add" %}
10745 ins_encode %{
10746 __ lock();
10747 __ addw($mem$$Address, $add$$constant);
10748 %}
10749 ins_pipe(pipe_cmpxchg);
10750 %}
10751
10752 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
10753 predicate(!n->as_LoadStore()->result_not_used());
10754 match(Set newval (GetAndAddS mem newval));
10755 effect(KILL cr);
10756 format %{ "xaddw_lock $mem, $newval\t# $newval -> short" %}
10757 ins_encode %{
10758 __ lock();
10759 __ xaddw($mem$$Address, $newval$$Register);
10760 __ narrow_subword_type($newval$$Register, T_SHORT);
10761 %}
10762 ins_pipe(pipe_cmpxchg);
10763 %}
10764
10765 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
10766 predicate(n->as_LoadStore()->result_not_used());
10767 match(Set dummy (GetAndAddI mem add));
10768 effect(KILL cr);
10769 format %{ "addl_lock $mem, $add" %}
10770 ins_encode %{
10771 __ lock();
10772 __ addl($mem$$Address, $add$$Register);
10773 %}
10774 ins_pipe(pipe_cmpxchg);
10775 %}
10776
10777 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
10778 predicate(n->as_LoadStore()->result_not_used());
10779 match(Set dummy (GetAndAddI mem add));
10780 effect(KILL cr);
10781 format %{ "addl_lock $mem, $add" %}
10782 ins_encode %{
10783 __ lock();
10784 __ addl($mem$$Address, $add$$constant);
10785 %}
10786 ins_pipe(pipe_cmpxchg);
10787 %}
10788
10789 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
10790 predicate(!n->as_LoadStore()->result_not_used());
10791 match(Set newval (GetAndAddI mem newval));
10792 effect(KILL cr);
10793 format %{ "xaddl_lock $mem, $newval" %}
10794 ins_encode %{
10795 __ lock();
10796 __ xaddl($mem$$Address, $newval$$Register);
10797 %}
10798 ins_pipe(pipe_cmpxchg);
10799 %}
10800
10801 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
10802 predicate(n->as_LoadStore()->result_not_used());
10803 match(Set dummy (GetAndAddL mem add));
10804 effect(KILL cr);
10805 format %{ "addq_lock $mem, $add" %}
10806 ins_encode %{
10807 __ lock();
10808 __ addq($mem$$Address, $add$$Register);
10809 %}
10810 ins_pipe(pipe_cmpxchg);
10811 %}
10812
10813 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
10814 predicate(n->as_LoadStore()->result_not_used());
10815 match(Set dummy (GetAndAddL mem add));
10816 effect(KILL cr);
10817 format %{ "addq_lock $mem, $add" %}
10818 ins_encode %{
10819 __ lock();
10820 __ addq($mem$$Address, $add$$constant);
10821 %}
10822 ins_pipe(pipe_cmpxchg);
10823 %}
10824
10825 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
10826 predicate(!n->as_LoadStore()->result_not_used());
10827 match(Set newval (GetAndAddL mem newval));
10828 effect(KILL cr);
10829 format %{ "xaddq_lock $mem, $newval" %}
10830 ins_encode %{
10831 __ lock();
10832 __ xaddq($mem$$Address, $newval$$Register);
10833 %}
10834 ins_pipe(pipe_cmpxchg);
10835 %}
10836
10837 instruct xchgB( memory mem, rRegI newval) %{
10838 match(Set newval (GetAndSetB mem newval));
10839 format %{ "XCHGB $newval,[$mem]\t# $newval -> byte" %}
10840 ins_encode %{
10841 __ xchgb($newval$$Register, $mem$$Address);
10842 __ narrow_subword_type($newval$$Register, T_BYTE);
10843 %}
10844 ins_pipe( pipe_cmpxchg );
10845 %}
10846
10847 instruct xchgS( memory mem, rRegI newval) %{
10848 match(Set newval (GetAndSetS mem newval));
10849 format %{ "XCHGW $newval,[$mem]\t# $newval -> short" %}
10850 ins_encode %{
10851 __ xchgw($newval$$Register, $mem$$Address);
10852 __ narrow_subword_type($newval$$Register, T_SHORT);
10853 %}
10854 ins_pipe( pipe_cmpxchg );
10855 %}
10856
10857 instruct xchgI( memory mem, rRegI newval) %{
10858 match(Set newval (GetAndSetI mem newval));
10859 format %{ "XCHGL $newval,[$mem]" %}
10860 ins_encode %{
10861 __ xchgl($newval$$Register, $mem$$Address);
10862 %}
10863 ins_pipe( pipe_cmpxchg );
10864 %}
10865
10866 instruct xchgL( memory mem, rRegL newval) %{
10867 match(Set newval (GetAndSetL mem newval));
10868 format %{ "XCHGL $newval,[$mem]" %}
10869 ins_encode %{
10870 __ xchgq($newval$$Register, $mem$$Address);
10871 %}
10872 ins_pipe( pipe_cmpxchg );
10873 %}
10874
10875 instruct xchgP( memory mem, rRegP newval) %{
10876 match(Set newval (GetAndSetP mem newval));
10877 predicate(n->as_LoadStore()->barrier_data() == 0);
10878 format %{ "XCHGQ $newval,[$mem]" %}
10879 ins_encode %{
10880 __ xchgq($newval$$Register, $mem$$Address);
10881 %}
10882 ins_pipe( pipe_cmpxchg );
10883 %}
10884
10885 instruct xchgN( memory mem, rRegN newval) %{
10886 predicate(n->as_LoadStore()->barrier_data() == 0);
10887 match(Set newval (GetAndSetN mem newval));
10888 format %{ "XCHGL $newval,$mem]" %}
10889 ins_encode %{
10890 __ xchgl($newval$$Register, $mem$$Address);
10891 %}
10892 ins_pipe( pipe_cmpxchg );
10893 %}
10894
10895 //----------Abs Instructions-------------------------------------------
10896
10897 // Integer Absolute Instructions
10898 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10899 %{
10900 match(Set dst (AbsI src));
10901 effect(TEMP dst, KILL cr);
10902 format %{ "xorl $dst, $dst\t# abs int\n\t"
10903 "subl $dst, $src\n\t"
10904 "cmovll $dst, $src" %}
10905 ins_encode %{
10906 __ xorl($dst$$Register, $dst$$Register);
10907 __ subl($dst$$Register, $src$$Register);
10908 __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
10909 %}
10910
10911 ins_pipe(ialu_reg_reg);
10912 %}
10913
10914 // Long Absolute Instructions
10915 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10916 %{
10917 match(Set dst (AbsL src));
10918 effect(TEMP dst, KILL cr);
10919 format %{ "xorl $dst, $dst\t# abs long\n\t"
10920 "subq $dst, $src\n\t"
10921 "cmovlq $dst, $src" %}
10922 ins_encode %{
10923 __ xorl($dst$$Register, $dst$$Register);
10924 __ subq($dst$$Register, $src$$Register);
10925 __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
10926 %}
10927
10928 ins_pipe(ialu_reg_reg);
10929 %}
10930
10931 //----------Subtraction Instructions-------------------------------------------
10932
10933 // Integer Subtraction Instructions
10934 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10935 %{
10936 predicate(!UseAPX);
10937 match(Set dst (SubI dst src));
10938 effect(KILL cr);
10939 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10940
10941 format %{ "subl $dst, $src\t# int" %}
10942 ins_encode %{
10943 __ subl($dst$$Register, $src$$Register);
10944 %}
10945 ins_pipe(ialu_reg_reg);
10946 %}
10947
10948 instruct subI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10949 %{
10950 predicate(UseAPX);
10951 match(Set dst (SubI src1 src2));
10952 effect(KILL cr);
10953 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
10954
10955 format %{ "esubl $dst, $src1, $src2\t# int ndd" %}
10956 ins_encode %{
10957 __ esubl($dst$$Register, $src1$$Register, $src2$$Register, false);
10958 %}
10959 ins_pipe(ialu_reg_reg);
10960 %}
10961
10962 instruct subI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10963 %{
10964 predicate(UseAPX);
10965 match(Set dst (SubI src1 src2));
10966 effect(KILL cr);
10967 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
10968
10969 format %{ "esubl $dst, $src1, $src2\t# int ndd" %}
10970 ins_encode %{
10971 __ esubl($dst$$Register, $src1$$Register, $src2$$constant, false);
10972 %}
10973 ins_pipe(ialu_reg_reg);
10974 %}
10975
10976 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10977 %{
10978 match(Set dst (SubI dst (LoadI src)));
10979 effect(KILL cr);
10980 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10981
10982 ins_cost(150);
10983 format %{ "subl $dst, $src\t# int" %}
10984 ins_encode %{
10985 __ subl($dst$$Register, $src$$Address);
10986 %}
10987 ins_pipe(ialu_reg_mem);
10988 %}
10989
10990 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10991 %{
10992 match(Set dst (StoreI dst (SubI (LoadI dst) src)));
10993 effect(KILL cr);
10994 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
10995
10996 ins_cost(150);
10997 format %{ "subl $dst, $src\t# int" %}
10998 ins_encode %{
10999 __ subl($dst$$Address, $src$$Register);
11000 %}
11001 ins_pipe(ialu_mem_reg);
11002 %}
11003
11004 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11005 %{
11006 predicate(!UseAPX);
11007 match(Set dst (SubL dst src));
11008 effect(KILL cr);
11009 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11010
11011 format %{ "subq $dst, $src\t# long" %}
11012 ins_encode %{
11013 __ subq($dst$$Register, $src$$Register);
11014 %}
11015 ins_pipe(ialu_reg_reg);
11016 %}
11017
11018 instruct subL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11019 %{
11020 predicate(UseAPX);
11021 match(Set dst (SubL src1 src2));
11022 effect(KILL cr);
11023 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11024
11025 format %{ "esubq $dst, $src1, $src2\t# long ndd" %}
11026 ins_encode %{
11027 __ esubq($dst$$Register, $src1$$Register, $src2$$Register, false);
11028 %}
11029 ins_pipe(ialu_reg_reg);
11030 %}
11031
11032 instruct subL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11033 %{
11034 predicate(UseAPX);
11035 match(Set dst (SubL src1 src2));
11036 effect(KILL cr);
11037 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11038
11039 format %{ "esubq $dst, $src1, $src2\t# long ndd" %}
11040 ins_encode %{
11041 __ esubq($dst$$Register, $src1$$Register, $src2$$constant, false);
11042 %}
11043 ins_pipe(ialu_reg_reg);
11044 %}
11045
11046 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11047 %{
11048 match(Set dst (SubL dst (LoadL src)));
11049 effect(KILL cr);
11050 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11051
11052 ins_cost(150);
11053 format %{ "subq $dst, $src\t# long" %}
11054 ins_encode %{
11055 __ subq($dst$$Register, $src$$Address);
11056 %}
11057 ins_pipe(ialu_reg_mem);
11058 %}
11059
11060 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11061 %{
11062 match(Set dst (StoreL dst (SubL (LoadL dst) src)));
11063 effect(KILL cr);
11064 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
11065
11066 ins_cost(150);
11067 format %{ "subq $dst, $src\t# long" %}
11068 ins_encode %{
11069 __ subq($dst$$Address, $src$$Register);
11070 %}
11071 ins_pipe(ialu_mem_reg);
11072 %}
11073
11074 // Subtract from a pointer
11075 // XXX hmpf???
11076 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
11077 %{
11078 match(Set dst (AddP dst (SubI zero src)));
11079 effect(KILL cr);
11080
11081 format %{ "subq $dst, $src\t# ptr - int" %}
11082 ins_encode %{
11083 __ subq($dst$$Register, $src$$Register);
11084 %}
11085 ins_pipe(ialu_reg_reg);
11086 %}
11087
11088 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
11089 %{
11090 predicate(!UseAPX);
11091 match(Set dst (SubI zero dst));
11092 effect(KILL cr);
11093 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11094
11095 format %{ "negl $dst\t# int" %}
11096 ins_encode %{
11097 __ negl($dst$$Register);
11098 %}
11099 ins_pipe(ialu_reg);
11100 %}
11101
11102 instruct negI_rReg_ndd(rRegI dst, rRegI src, immI_0 zero, rFlagsReg cr)
11103 %{
11104 predicate(UseAPX);
11105 match(Set dst (SubI zero src));
11106 effect(KILL cr);
11107 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr2);
11108
11109 format %{ "enegl $dst, $src\t# int ndd" %}
11110 ins_encode %{
11111 __ enegl($dst$$Register, $src$$Register, false);
11112 %}
11113 ins_pipe(ialu_reg);
11114 %}
11115
11116 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
11117 %{
11118 predicate(!UseAPX);
11119 match(Set dst (NegI dst));
11120 effect(KILL cr);
11121 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11122
11123 format %{ "negl $dst\t# int" %}
11124 ins_encode %{
11125 __ negl($dst$$Register);
11126 %}
11127 ins_pipe(ialu_reg);
11128 %}
11129
11130 instruct negI_rReg_2_ndd(rRegI dst, rRegI src, rFlagsReg cr)
11131 %{
11132 predicate(UseAPX);
11133 match(Set dst (NegI src));
11134 effect(KILL cr);
11135 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11136
11137 format %{ "enegl $dst, $src\t# int ndd" %}
11138 ins_encode %{
11139 __ enegl($dst$$Register, $src$$Register, false);
11140 %}
11141 ins_pipe(ialu_reg);
11142 %}
11143
11144 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
11145 %{
11146 match(Set dst (StoreI dst (SubI zero (LoadI dst))));
11147 effect(KILL cr);
11148 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11149
11150 format %{ "negl $dst\t# int" %}
11151 ins_encode %{
11152 __ negl($dst$$Address);
11153 %}
11154 ins_pipe(ialu_reg);
11155 %}
11156
11157 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
11158 %{
11159 predicate(!UseAPX);
11160 match(Set dst (SubL zero dst));
11161 effect(KILL cr);
11162 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11163
11164 format %{ "negq $dst\t# long" %}
11165 ins_encode %{
11166 __ negq($dst$$Register);
11167 %}
11168 ins_pipe(ialu_reg);
11169 %}
11170
11171 instruct negL_rReg_ndd(rRegL dst, rRegL src, immL0 zero, rFlagsReg cr)
11172 %{
11173 predicate(UseAPX);
11174 match(Set dst (SubL zero src));
11175 effect(KILL cr);
11176 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr2);
11177
11178 format %{ "enegq $dst, $src\t# long ndd" %}
11179 ins_encode %{
11180 __ enegq($dst$$Register, $src$$Register, false);
11181 %}
11182 ins_pipe(ialu_reg);
11183 %}
11184
11185 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
11186 %{
11187 predicate(!UseAPX);
11188 match(Set dst (NegL dst));
11189 effect(KILL cr);
11190 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11191
11192 format %{ "negq $dst\t# int" %}
11193 ins_encode %{
11194 __ negq($dst$$Register);
11195 %}
11196 ins_pipe(ialu_reg);
11197 %}
11198
11199 instruct negL_rReg_2_ndd(rRegL dst, rRegL src, rFlagsReg cr)
11200 %{
11201 predicate(UseAPX);
11202 match(Set dst (NegL src));
11203 effect(KILL cr);
11204 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_ndd_demotable_opr1);
11205
11206 format %{ "enegq $dst, $src\t# long ndd" %}
11207 ins_encode %{
11208 __ enegq($dst$$Register, $src$$Register, false);
11209 %}
11210 ins_pipe(ialu_reg);
11211 %}
11212
11213 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
11214 %{
11215 match(Set dst (StoreL dst (SubL zero (LoadL dst))));
11216 effect(KILL cr);
11217 flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
11218
11219 format %{ "negq $dst\t# long" %}
11220 ins_encode %{
11221 __ negq($dst$$Address);
11222 %}
11223 ins_pipe(ialu_reg);
11224 %}
11225
11226 //----------Multiplication/Division Instructions-------------------------------
11227 // Integer Multiplication Instructions
11228 // Multiply Register
11229
11230 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
11231 %{
11232 predicate(!UseAPX);
11233 match(Set dst (MulI dst src));
11234 effect(KILL cr);
11235
11236 ins_cost(300);
11237 format %{ "imull $dst, $src\t# int" %}
11238 ins_encode %{
11239 __ imull($dst$$Register, $src$$Register);
11240 %}
11241 ins_pipe(ialu_reg_reg_alu0);
11242 %}
11243
11244 instruct mulI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
11245 %{
11246 predicate(UseAPX);
11247 match(Set dst (MulI src1 src2));
11248 effect(KILL cr);
11249 flag(PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
11250
11251 ins_cost(300);
11252 format %{ "eimull $dst, $src1, $src2\t# int ndd" %}
11253 ins_encode %{
11254 __ eimull($dst$$Register, $src1$$Register, $src2$$Register, false);
11255 %}
11256 ins_pipe(ialu_reg_reg_alu0);
11257 %}
11258
11259 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
11260 %{
11261 match(Set dst (MulI src imm));
11262 effect(KILL cr);
11263
11264 ins_cost(300);
11265 format %{ "imull $dst, $src, $imm\t# int" %}
11266 ins_encode %{
11267 __ imull($dst$$Register, $src$$Register, $imm$$constant);
11268 %}
11269 ins_pipe(ialu_reg_reg_alu0);
11270 %}
11271
11272 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
11273 %{
11274 match(Set dst (MulI dst (LoadI src)));
11275 effect(KILL cr);
11276
11277 ins_cost(350);
11278 format %{ "imull $dst, $src\t# int" %}
11279 ins_encode %{
11280 __ imull($dst$$Register, $src$$Address);
11281 %}
11282 ins_pipe(ialu_reg_mem_alu0);
11283 %}
11284
11285 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
11286 %{
11287 match(Set dst (MulI (LoadI src) imm));
11288 effect(KILL cr);
11289
11290 ins_cost(300);
11291 format %{ "imull $dst, $src, $imm\t# int" %}
11292 ins_encode %{
11293 __ imull($dst$$Register, $src$$Address, $imm$$constant);
11294 %}
11295 ins_pipe(ialu_reg_mem_alu0);
11296 %}
11297
11298 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
11299 %{
11300 match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
11301 effect(KILL cr, KILL src2);
11302
11303 expand %{ mulI_rReg(dst, src1, cr);
11304 mulI_rReg(src2, src3, cr);
11305 addI_rReg(dst, src2, cr); %}
11306 %}
11307
11308 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11309 %{
11310 predicate(!UseAPX);
11311 match(Set dst (MulL dst src));
11312 effect(KILL cr);
11313
11314 ins_cost(300);
11315 format %{ "imulq $dst, $src\t# long" %}
11316 ins_encode %{
11317 __ imulq($dst$$Register, $src$$Register);
11318 %}
11319 ins_pipe(ialu_reg_reg_alu0);
11320 %}
11321
11322 instruct mulL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11323 %{
11324 predicate(UseAPX);
11325 match(Set dst (MulL src1 src2));
11326 effect(KILL cr);
11327 flag(PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
11328
11329 ins_cost(300);
11330 format %{ "eimulq $dst, $src1, $src2\t# long ndd" %}
11331 ins_encode %{
11332 __ eimulq($dst$$Register, $src1$$Register, $src2$$Register, false);
11333 %}
11334 ins_pipe(ialu_reg_reg_alu0);
11335 %}
11336
11337 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
11338 %{
11339 match(Set dst (MulL src imm));
11340 effect(KILL cr);
11341
11342 ins_cost(300);
11343 format %{ "imulq $dst, $src, $imm\t# long" %}
11344 ins_encode %{
11345 __ imulq($dst$$Register, $src$$Register, $imm$$constant);
11346 %}
11347 ins_pipe(ialu_reg_reg_alu0);
11348 %}
11349
11350 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
11351 %{
11352 match(Set dst (MulL dst (LoadL src)));
11353 effect(KILL cr);
11354
11355 ins_cost(350);
11356 format %{ "imulq $dst, $src\t# long" %}
11357 ins_encode %{
11358 __ imulq($dst$$Register, $src$$Address);
11359 %}
11360 ins_pipe(ialu_reg_mem_alu0);
11361 %}
11362
11363
11364 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
11365 %{
11366 match(Set dst (MulL (LoadL src) imm));
11367 effect(KILL cr);
11368
11369 ins_cost(300);
11370 format %{ "imulq $dst, $src, $imm\t# long" %}
11371 ins_encode %{
11372 __ imulq($dst$$Register, $src$$Address, $imm$$constant);
11373 %}
11374 ins_pipe(ialu_reg_mem_alu0);
11375 %}
11376
11377 instruct mulHiLoL_rReg(rax_RegL rax, rdx_RegL rdx, rRegL src, rFlagsReg cr)
11378 %{
11379 match(MulHiLoL src rax);
11380 match(MulHiLoL rax src);
11381 effect(KILL cr);
11382
11383 ins_cost(300);
11384 format %{ "imulq RDX:RAX, RAX, $src\t# mulhilo" %}
11385 ins_encode %{
11386 __ imulq($src$$Register);
11387 %}
11388 ins_pipe(ialu_reg_reg_alu0);
11389 %}
11390
11391 instruct umulHiLoL_rReg(rax_RegL rax, rdx_RegL rdx, rRegL src, rFlagsReg cr)
11392 %{
11393 match(UMulHiLoL src rax);
11394 match(UMulHiLoL rax src);
11395 effect(KILL cr);
11396
11397 ins_cost(300);
11398 format %{ "mulq RDX:RAX, RAX, $src\t# umulhilo" %}
11399 ins_encode %{
11400 __ mulq($src$$Register);
11401 %}
11402 ins_pipe(ialu_reg_reg_alu0);
11403 %}
11404
11405 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
11406 %{
11407 match(Set dst (MulHiL src rax));
11408 effect(USE_KILL rax, KILL cr);
11409
11410 ins_cost(300);
11411 format %{ "imulq RDX:RAX, RAX, $src\t# mulhi" %}
11412 ins_encode %{
11413 __ imulq($src$$Register);
11414 %}
11415 ins_pipe(ialu_reg_reg_alu0);
11416 %}
11417
11418 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
11419 %{
11420 match(Set dst (UMulHiL src rax));
11421 effect(USE_KILL rax, KILL cr);
11422
11423 ins_cost(300);
11424 format %{ "mulq RDX:RAX, RAX, $src\t# umulhi" %}
11425 ins_encode %{
11426 __ mulq($src$$Register);
11427 %}
11428 ins_pipe(ialu_reg_reg_alu0);
11429 %}
11430
11431 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
11432 rFlagsReg cr)
11433 %{
11434 match(Set rax (DivI rax div));
11435 effect(KILL rdx, KILL cr);
11436
11437 ins_cost(30*100+10*100); // XXX
11438 format %{ "cmpl rax, 0x80000000\t# idiv\n\t"
11439 "jne,s normal\n\t"
11440 "xorl rdx, rdx\n\t"
11441 "cmpl $div, -1\n\t"
11442 "je,s done\n"
11443 "normal: cdql\n\t"
11444 "idivl $div\n"
11445 "done:" %}
11446 ins_encode(cdql_enc(div));
11447 ins_pipe(ialu_reg_reg_alu0);
11448 %}
11449
11450 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
11451 rFlagsReg cr)
11452 %{
11453 match(Set rax (DivL rax div));
11454 effect(KILL rdx, KILL cr);
11455
11456 ins_cost(30*100+10*100); // XXX
11457 format %{ "movq rdx, 0x8000000000000000\t# ldiv\n\t"
11458 "cmpq rax, rdx\n\t"
11459 "jne,s normal\n\t"
11460 "xorl rdx, rdx\n\t"
11461 "cmpq $div, -1\n\t"
11462 "je,s done\n"
11463 "normal: cdqq\n\t"
11464 "idivq $div\n"
11465 "done:" %}
11466 ins_encode(cdqq_enc(div));
11467 ins_pipe(ialu_reg_reg_alu0);
11468 %}
11469
11470 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
11471 %{
11472 match(Set rax (UDivI rax div));
11473 effect(KILL rdx, KILL cr);
11474
11475 ins_cost(300);
11476 format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
11477 ins_encode %{
11478 __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
11479 %}
11480 ins_pipe(ialu_reg_reg_alu0);
11481 %}
11482
11483 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
11484 %{
11485 match(Set rax (UDivL rax div));
11486 effect(KILL rdx, KILL cr);
11487
11488 ins_cost(300);
11489 format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
11490 ins_encode %{
11491 __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
11492 %}
11493 ins_pipe(ialu_reg_reg_alu0);
11494 %}
11495
11496 // Integer DIVMOD with Register, both quotient and mod results
11497 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
11498 rFlagsReg cr)
11499 %{
11500 match(DivModI rax div);
11501 effect(KILL cr);
11502
11503 ins_cost(30*100+10*100); // XXX
11504 format %{ "cmpl rax, 0x80000000\t# idiv\n\t"
11505 "jne,s normal\n\t"
11506 "xorl rdx, rdx\n\t"
11507 "cmpl $div, -1\n\t"
11508 "je,s done\n"
11509 "normal: cdql\n\t"
11510 "idivl $div\n"
11511 "done:" %}
11512 ins_encode(cdql_enc(div));
11513 ins_pipe(pipe_slow);
11514 %}
11515
11516 // Long DIVMOD with Register, both quotient and mod results
11517 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
11518 rFlagsReg cr)
11519 %{
11520 match(DivModL rax div);
11521 effect(KILL cr);
11522
11523 ins_cost(30*100+10*100); // XXX
11524 format %{ "movq rdx, 0x8000000000000000\t# ldiv\n\t"
11525 "cmpq rax, rdx\n\t"
11526 "jne,s normal\n\t"
11527 "xorl rdx, rdx\n\t"
11528 "cmpq $div, -1\n\t"
11529 "je,s done\n"
11530 "normal: cdqq\n\t"
11531 "idivq $div\n"
11532 "done:" %}
11533 ins_encode(cdqq_enc(div));
11534 ins_pipe(pipe_slow);
11535 %}
11536
11537 // Unsigned integer DIVMOD with Register, both quotient and mod results
11538 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
11539 no_rax_rdx_RegI div, rFlagsReg cr)
11540 %{
11541 match(UDivModI rax div);
11542 effect(TEMP tmp, KILL cr);
11543
11544 ins_cost(300);
11545 format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
11546 "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
11547 %}
11548 ins_encode %{
11549 __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
11550 %}
11551 ins_pipe(pipe_slow);
11552 %}
11553
11554 // Unsigned long DIVMOD with Register, both quotient and mod results
11555 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
11556 no_rax_rdx_RegL div, rFlagsReg cr)
11557 %{
11558 match(UDivModL rax div);
11559 effect(TEMP tmp, KILL cr);
11560
11561 ins_cost(300);
11562 format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
11563 "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
11564 %}
11565 ins_encode %{
11566 __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
11567 %}
11568 ins_pipe(pipe_slow);
11569 %}
11570
11571 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
11572 rFlagsReg cr)
11573 %{
11574 match(Set rdx (ModI rax div));
11575 effect(KILL rax, KILL cr);
11576
11577 ins_cost(300); // XXX
11578 format %{ "cmpl rax, 0x80000000\t# irem\n\t"
11579 "jne,s normal\n\t"
11580 "xorl rdx, rdx\n\t"
11581 "cmpl $div, -1\n\t"
11582 "je,s done\n"
11583 "normal: cdql\n\t"
11584 "idivl $div\n"
11585 "done:" %}
11586 ins_encode(cdql_enc(div));
11587 ins_pipe(ialu_reg_reg_alu0);
11588 %}
11589
11590 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
11591 rFlagsReg cr)
11592 %{
11593 match(Set rdx (ModL rax div));
11594 effect(KILL rax, KILL cr);
11595
11596 ins_cost(300); // XXX
11597 format %{ "movq rdx, 0x8000000000000000\t# lrem\n\t"
11598 "cmpq rax, rdx\n\t"
11599 "jne,s normal\n\t"
11600 "xorl rdx, rdx\n\t"
11601 "cmpq $div, -1\n\t"
11602 "je,s done\n"
11603 "normal: cdqq\n\t"
11604 "idivq $div\n"
11605 "done:" %}
11606 ins_encode(cdqq_enc(div));
11607 ins_pipe(ialu_reg_reg_alu0);
11608 %}
11609
11610 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
11611 %{
11612 match(Set rdx (UModI rax div));
11613 effect(KILL rax, KILL cr);
11614
11615 ins_cost(300);
11616 format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
11617 ins_encode %{
11618 __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
11619 %}
11620 ins_pipe(ialu_reg_reg_alu0);
11621 %}
11622
11623 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
11624 %{
11625 match(Set rdx (UModL rax div));
11626 effect(KILL rax, KILL cr);
11627
11628 ins_cost(300);
11629 format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
11630 ins_encode %{
11631 __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
11632 %}
11633 ins_pipe(ialu_reg_reg_alu0);
11634 %}
11635
11636 // Integer Shift Instructions
11637 // Shift Left by one, two, three
11638 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
11639 %{
11640 predicate(!UseAPX);
11641 match(Set dst (LShiftI dst shift));
11642 effect(KILL cr);
11643
11644 format %{ "sall $dst, $shift" %}
11645 ins_encode %{
11646 __ sall($dst$$Register, $shift$$constant);
11647 %}
11648 ins_pipe(ialu_reg);
11649 %}
11650
11651 // Shift Left by one, two, three
11652 instruct salI_rReg_immI2_ndd(rRegI dst, rRegI src, immI2 shift, rFlagsReg cr)
11653 %{
11654 predicate(UseAPX);
11655 match(Set dst (LShiftI src shift));
11656 effect(KILL cr);
11657 flag(PD::Flag_ndd_demotable_opr1);
11658
11659 format %{ "esall $dst, $src, $shift\t# int(ndd)" %}
11660 ins_encode %{
11661 __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
11662 %}
11663 ins_pipe(ialu_reg);
11664 %}
11665
11666 // Shift Left by 8-bit immediate
11667 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
11668 %{
11669 predicate(!UseAPX);
11670 match(Set dst (LShiftI dst shift));
11671 effect(KILL cr);
11672
11673 format %{ "sall $dst, $shift" %}
11674 ins_encode %{
11675 __ sall($dst$$Register, $shift$$constant);
11676 %}
11677 ins_pipe(ialu_reg);
11678 %}
11679
11680 // Shift Left by 8-bit immediate
11681 instruct salI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
11682 %{
11683 predicate(UseAPX);
11684 match(Set dst (LShiftI src shift));
11685 effect(KILL cr);
11686 flag(PD::Flag_ndd_demotable_opr1);
11687
11688 format %{ "esall $dst, $src, $shift\t# int (ndd)" %}
11689 ins_encode %{
11690 __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
11691 %}
11692 ins_pipe(ialu_reg);
11693 %}
11694
11695 // Shift Left by 8-bit immediate
11696 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
11697 %{
11698 match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
11699 effect(KILL cr);
11700
11701 format %{ "sall $dst, $shift" %}
11702 ins_encode %{
11703 __ sall($dst$$Address, $shift$$constant);
11704 %}
11705 ins_pipe(ialu_mem_imm);
11706 %}
11707
11708 // Shift Left by variable
11709 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
11710 %{
11711 predicate(!VM_Version::supports_bmi2());
11712 match(Set dst (LShiftI dst shift));
11713 effect(KILL cr);
11714
11715 format %{ "sall $dst, $shift" %}
11716 ins_encode %{
11717 __ sall($dst$$Register);
11718 %}
11719 ins_pipe(ialu_reg_reg);
11720 %}
11721
11722 // Shift Left by variable
11723 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
11724 %{
11725 predicate(!VM_Version::supports_bmi2());
11726 match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
11727 effect(KILL cr);
11728
11729 format %{ "sall $dst, $shift" %}
11730 ins_encode %{
11731 __ sall($dst$$Address);
11732 %}
11733 ins_pipe(ialu_mem_reg);
11734 %}
11735
11736 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
11737 %{
11738 predicate(VM_Version::supports_bmi2());
11739 match(Set dst (LShiftI src shift));
11740
11741 format %{ "shlxl $dst, $src, $shift" %}
11742 ins_encode %{
11743 __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
11744 %}
11745 ins_pipe(ialu_reg_reg);
11746 %}
11747
11748 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
11749 %{
11750 predicate(VM_Version::supports_bmi2());
11751 match(Set dst (LShiftI (LoadI src) shift));
11752 ins_cost(175);
11753 format %{ "shlxl $dst, $src, $shift" %}
11754 ins_encode %{
11755 __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
11756 %}
11757 ins_pipe(ialu_reg_mem);
11758 %}
11759
11760 // Arithmetic Shift Right by 8-bit immediate
11761 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
11762 %{
11763 predicate(!UseAPX);
11764 match(Set dst (RShiftI dst shift));
11765 effect(KILL cr);
11766
11767 format %{ "sarl $dst, $shift" %}
11768 ins_encode %{
11769 __ sarl($dst$$Register, $shift$$constant);
11770 %}
11771 ins_pipe(ialu_mem_imm);
11772 %}
11773
11774 // Arithmetic Shift Right by 8-bit immediate
11775 instruct sarI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
11776 %{
11777 predicate(UseAPX);
11778 match(Set dst (RShiftI src shift));
11779 effect(KILL cr);
11780 flag(PD::Flag_ndd_demotable_opr1);
11781
11782 format %{ "esarl $dst, $src, $shift\t# int (ndd)" %}
11783 ins_encode %{
11784 __ esarl($dst$$Register, $src$$Register, $shift$$constant, false);
11785 %}
11786 ins_pipe(ialu_mem_imm);
11787 %}
11788
11789 // Arithmetic Shift Right by 8-bit immediate
11790 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
11791 %{
11792 match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
11793 effect(KILL cr);
11794
11795 format %{ "sarl $dst, $shift" %}
11796 ins_encode %{
11797 __ sarl($dst$$Address, $shift$$constant);
11798 %}
11799 ins_pipe(ialu_mem_imm);
11800 %}
11801
11802 // Arithmetic Shift Right by variable
11803 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
11804 %{
11805 predicate(!VM_Version::supports_bmi2());
11806 match(Set dst (RShiftI dst shift));
11807 effect(KILL cr);
11808
11809 format %{ "sarl $dst, $shift" %}
11810 ins_encode %{
11811 __ sarl($dst$$Register);
11812 %}
11813 ins_pipe(ialu_reg_reg);
11814 %}
11815
11816 // Arithmetic Shift Right by variable
11817 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
11818 %{
11819 predicate(!VM_Version::supports_bmi2());
11820 match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
11821 effect(KILL cr);
11822
11823 format %{ "sarl $dst, $shift" %}
11824 ins_encode %{
11825 __ sarl($dst$$Address);
11826 %}
11827 ins_pipe(ialu_mem_reg);
11828 %}
11829
11830 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
11831 %{
11832 predicate(VM_Version::supports_bmi2());
11833 match(Set dst (RShiftI src shift));
11834
11835 format %{ "sarxl $dst, $src, $shift" %}
11836 ins_encode %{
11837 __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
11838 %}
11839 ins_pipe(ialu_reg_reg);
11840 %}
11841
11842 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
11843 %{
11844 predicate(VM_Version::supports_bmi2());
11845 match(Set dst (RShiftI (LoadI src) shift));
11846 ins_cost(175);
11847 format %{ "sarxl $dst, $src, $shift" %}
11848 ins_encode %{
11849 __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
11850 %}
11851 ins_pipe(ialu_reg_mem);
11852 %}
11853
11854 // Logical Shift Right by 8-bit immediate
11855 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
11856 %{
11857 predicate(!UseAPX);
11858 match(Set dst (URShiftI dst shift));
11859 effect(KILL cr);
11860
11861 format %{ "shrl $dst, $shift" %}
11862 ins_encode %{
11863 __ shrl($dst$$Register, $shift$$constant);
11864 %}
11865 ins_pipe(ialu_reg);
11866 %}
11867
11868 // Logical Shift Right by 8-bit immediate
11869 instruct shrI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
11870 %{
11871 predicate(UseAPX);
11872 match(Set dst (URShiftI src shift));
11873 effect(KILL cr);
11874 flag(PD::Flag_ndd_demotable_opr1);
11875
11876 format %{ "eshrl $dst, $src, $shift\t # int (ndd)" %}
11877 ins_encode %{
11878 __ eshrl($dst$$Register, $src$$Register, $shift$$constant, false);
11879 %}
11880 ins_pipe(ialu_reg);
11881 %}
11882
11883 // Logical Shift Right by 8-bit immediate
11884 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
11885 %{
11886 match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
11887 effect(KILL cr);
11888
11889 format %{ "shrl $dst, $shift" %}
11890 ins_encode %{
11891 __ shrl($dst$$Address, $shift$$constant);
11892 %}
11893 ins_pipe(ialu_mem_imm);
11894 %}
11895
11896 // Logical Shift Right by variable
11897 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
11898 %{
11899 predicate(!VM_Version::supports_bmi2());
11900 match(Set dst (URShiftI dst shift));
11901 effect(KILL cr);
11902
11903 format %{ "shrl $dst, $shift" %}
11904 ins_encode %{
11905 __ shrl($dst$$Register);
11906 %}
11907 ins_pipe(ialu_reg_reg);
11908 %}
11909
11910 // Logical Shift Right by variable
11911 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
11912 %{
11913 predicate(!VM_Version::supports_bmi2());
11914 match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
11915 effect(KILL cr);
11916
11917 format %{ "shrl $dst, $shift" %}
11918 ins_encode %{
11919 __ shrl($dst$$Address);
11920 %}
11921 ins_pipe(ialu_mem_reg);
11922 %}
11923
11924 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
11925 %{
11926 predicate(VM_Version::supports_bmi2());
11927 match(Set dst (URShiftI src shift));
11928
11929 format %{ "shrxl $dst, $src, $shift" %}
11930 ins_encode %{
11931 __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
11932 %}
11933 ins_pipe(ialu_reg_reg);
11934 %}
11935
11936 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
11937 %{
11938 predicate(VM_Version::supports_bmi2());
11939 match(Set dst (URShiftI (LoadI src) shift));
11940 ins_cost(175);
11941 format %{ "shrxl $dst, $src, $shift" %}
11942 ins_encode %{
11943 __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
11944 %}
11945 ins_pipe(ialu_reg_mem);
11946 %}
11947
11948 // Long Shift Instructions
11949 // Shift Left by one, two, three
11950 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
11951 %{
11952 predicate(!UseAPX);
11953 match(Set dst (LShiftL dst shift));
11954 effect(KILL cr);
11955
11956 format %{ "salq $dst, $shift" %}
11957 ins_encode %{
11958 __ salq($dst$$Register, $shift$$constant);
11959 %}
11960 ins_pipe(ialu_reg);
11961 %}
11962
11963 // Shift Left by one, two, three
11964 instruct salL_rReg_immI2_ndd(rRegL dst, rRegL src, immI2 shift, rFlagsReg cr)
11965 %{
11966 predicate(UseAPX);
11967 match(Set dst (LShiftL src shift));
11968 effect(KILL cr);
11969 flag(PD::Flag_ndd_demotable_opr1);
11970
11971 format %{ "esalq $dst, $src, $shift\t# long (ndd)" %}
11972 ins_encode %{
11973 __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
11974 %}
11975 ins_pipe(ialu_reg);
11976 %}
11977
11978 // Shift Left by 8-bit immediate
11979 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
11980 %{
11981 predicate(!UseAPX);
11982 match(Set dst (LShiftL dst shift));
11983 effect(KILL cr);
11984
11985 format %{ "salq $dst, $shift" %}
11986 ins_encode %{
11987 __ salq($dst$$Register, $shift$$constant);
11988 %}
11989 ins_pipe(ialu_reg);
11990 %}
11991
11992 // Shift Left by 8-bit immediate
11993 instruct salL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
11994 %{
11995 predicate(UseAPX);
11996 match(Set dst (LShiftL src shift));
11997 effect(KILL cr);
11998 flag(PD::Flag_ndd_demotable_opr1);
11999
12000 format %{ "esalq $dst, $src, $shift\t# long (ndd)" %}
12001 ins_encode %{
12002 __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
12003 %}
12004 ins_pipe(ialu_reg);
12005 %}
12006
12007 // Shift Left by 8-bit immediate
12008 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
12009 %{
12010 match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
12011 effect(KILL cr);
12012
12013 format %{ "salq $dst, $shift" %}
12014 ins_encode %{
12015 __ salq($dst$$Address, $shift$$constant);
12016 %}
12017 ins_pipe(ialu_mem_imm);
12018 %}
12019
12020 // Shift Left by variable
12021 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12022 %{
12023 predicate(!VM_Version::supports_bmi2());
12024 match(Set dst (LShiftL dst shift));
12025 effect(KILL cr);
12026
12027 format %{ "salq $dst, $shift" %}
12028 ins_encode %{
12029 __ salq($dst$$Register);
12030 %}
12031 ins_pipe(ialu_reg_reg);
12032 %}
12033
12034 // Shift Left by variable
12035 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
12036 %{
12037 predicate(!VM_Version::supports_bmi2());
12038 match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
12039 effect(KILL cr);
12040
12041 format %{ "salq $dst, $shift" %}
12042 ins_encode %{
12043 __ salq($dst$$Address);
12044 %}
12045 ins_pipe(ialu_mem_reg);
12046 %}
12047
12048 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
12049 %{
12050 predicate(VM_Version::supports_bmi2());
12051 match(Set dst (LShiftL src shift));
12052
12053 format %{ "shlxq $dst, $src, $shift" %}
12054 ins_encode %{
12055 __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
12056 %}
12057 ins_pipe(ialu_reg_reg);
12058 %}
12059
12060 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
12061 %{
12062 predicate(VM_Version::supports_bmi2());
12063 match(Set dst (LShiftL (LoadL src) shift));
12064 ins_cost(175);
12065 format %{ "shlxq $dst, $src, $shift" %}
12066 ins_encode %{
12067 __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
12068 %}
12069 ins_pipe(ialu_reg_mem);
12070 %}
12071
12072 // Arithmetic Shift Right by 8-bit immediate
12073 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
12074 %{
12075 predicate(!UseAPX);
12076 match(Set dst (RShiftL dst shift));
12077 effect(KILL cr);
12078
12079 format %{ "sarq $dst, $shift" %}
12080 ins_encode %{
12081 __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
12082 %}
12083 ins_pipe(ialu_mem_imm);
12084 %}
12085
12086 // Arithmetic Shift Right by 8-bit immediate
12087 instruct sarL_rReg_imm_ndd(rRegL dst, rRegL src, immI shift, rFlagsReg cr)
12088 %{
12089 predicate(UseAPX);
12090 match(Set dst (RShiftL src shift));
12091 effect(KILL cr);
12092 flag(PD::Flag_ndd_demotable_opr1);
12093
12094 format %{ "esarq $dst, $src, $shift\t# long (ndd)" %}
12095 ins_encode %{
12096 __ esarq($dst$$Register, $src$$Register, (unsigned char)($shift$$constant & 0x3F), false);
12097 %}
12098 ins_pipe(ialu_mem_imm);
12099 %}
12100
12101 // Arithmetic Shift Right by 8-bit immediate
12102 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
12103 %{
12104 match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
12105 effect(KILL cr);
12106
12107 format %{ "sarq $dst, $shift" %}
12108 ins_encode %{
12109 __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
12110 %}
12111 ins_pipe(ialu_mem_imm);
12112 %}
12113
12114 // Arithmetic Shift Right by variable
12115 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12116 %{
12117 predicate(!VM_Version::supports_bmi2());
12118 match(Set dst (RShiftL dst shift));
12119 effect(KILL cr);
12120
12121 format %{ "sarq $dst, $shift" %}
12122 ins_encode %{
12123 __ sarq($dst$$Register);
12124 %}
12125 ins_pipe(ialu_reg_reg);
12126 %}
12127
12128 // Arithmetic Shift Right by variable
12129 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
12130 %{
12131 predicate(!VM_Version::supports_bmi2());
12132 match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
12133 effect(KILL cr);
12134
12135 format %{ "sarq $dst, $shift" %}
12136 ins_encode %{
12137 __ sarq($dst$$Address);
12138 %}
12139 ins_pipe(ialu_mem_reg);
12140 %}
12141
12142 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
12143 %{
12144 predicate(VM_Version::supports_bmi2());
12145 match(Set dst (RShiftL src shift));
12146
12147 format %{ "sarxq $dst, $src, $shift" %}
12148 ins_encode %{
12149 __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
12150 %}
12151 ins_pipe(ialu_reg_reg);
12152 %}
12153
12154 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
12155 %{
12156 predicate(VM_Version::supports_bmi2());
12157 match(Set dst (RShiftL (LoadL src) shift));
12158 ins_cost(175);
12159 format %{ "sarxq $dst, $src, $shift" %}
12160 ins_encode %{
12161 __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
12162 %}
12163 ins_pipe(ialu_reg_mem);
12164 %}
12165
12166 // Logical Shift Right by 8-bit immediate
12167 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
12168 %{
12169 predicate(!UseAPX);
12170 match(Set dst (URShiftL dst shift));
12171 effect(KILL cr);
12172
12173 format %{ "shrq $dst, $shift" %}
12174 ins_encode %{
12175 __ shrq($dst$$Register, $shift$$constant);
12176 %}
12177 ins_pipe(ialu_reg);
12178 %}
12179
12180 // Logical Shift Right by 8-bit immediate
12181 instruct shrL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
12182 %{
12183 predicate(UseAPX);
12184 match(Set dst (URShiftL src shift));
12185 effect(KILL cr);
12186 flag(PD::Flag_ndd_demotable_opr1);
12187
12188 format %{ "eshrq $dst, $src, $shift\t# long (ndd)" %}
12189 ins_encode %{
12190 __ eshrq($dst$$Register, $src$$Register, $shift$$constant, false);
12191 %}
12192 ins_pipe(ialu_reg);
12193 %}
12194
12195 // Logical Shift Right by 8-bit immediate
12196 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
12197 %{
12198 match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
12199 effect(KILL cr);
12200
12201 format %{ "shrq $dst, $shift" %}
12202 ins_encode %{
12203 __ shrq($dst$$Address, $shift$$constant);
12204 %}
12205 ins_pipe(ialu_mem_imm);
12206 %}
12207
12208 // Logical Shift Right by variable
12209 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12210 %{
12211 predicate(!VM_Version::supports_bmi2());
12212 match(Set dst (URShiftL dst shift));
12213 effect(KILL cr);
12214
12215 format %{ "shrq $dst, $shift" %}
12216 ins_encode %{
12217 __ shrq($dst$$Register);
12218 %}
12219 ins_pipe(ialu_reg_reg);
12220 %}
12221
12222 // Logical Shift Right by variable
12223 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
12224 %{
12225 predicate(!VM_Version::supports_bmi2());
12226 match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
12227 effect(KILL cr);
12228
12229 format %{ "shrq $dst, $shift" %}
12230 ins_encode %{
12231 __ shrq($dst$$Address);
12232 %}
12233 ins_pipe(ialu_mem_reg);
12234 %}
12235
12236 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
12237 %{
12238 predicate(VM_Version::supports_bmi2());
12239 match(Set dst (URShiftL src shift));
12240
12241 format %{ "shrxq $dst, $src, $shift" %}
12242 ins_encode %{
12243 __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
12244 %}
12245 ins_pipe(ialu_reg_reg);
12246 %}
12247
12248 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
12249 %{
12250 predicate(VM_Version::supports_bmi2());
12251 match(Set dst (URShiftL (LoadL src) shift));
12252 ins_cost(175);
12253 format %{ "shrxq $dst, $src, $shift" %}
12254 ins_encode %{
12255 __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
12256 %}
12257 ins_pipe(ialu_reg_mem);
12258 %}
12259
12260 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
12261 // This idiom is used by the compiler for the i2b bytecode.
12262 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
12263 %{
12264 match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
12265
12266 format %{ "movsbl $dst, $src\t# i2b" %}
12267 ins_encode %{
12268 __ movsbl($dst$$Register, $src$$Register);
12269 %}
12270 ins_pipe(ialu_reg_reg);
12271 %}
12272
12273 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
12274 // This idiom is used by the compiler the i2s bytecode.
12275 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
12276 %{
12277 match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
12278
12279 format %{ "movswl $dst, $src\t# i2s" %}
12280 ins_encode %{
12281 __ movswl($dst$$Register, $src$$Register);
12282 %}
12283 ins_pipe(ialu_reg_reg);
12284 %}
12285
12286 // ROL/ROR instructions
12287
12288 // Rotate left by constant.
12289 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
12290 %{
12291 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12292 match(Set dst (RotateLeft dst shift));
12293 effect(KILL cr);
12294 format %{ "roll $dst, $shift" %}
12295 ins_encode %{
12296 __ roll($dst$$Register, $shift$$constant);
12297 %}
12298 ins_pipe(ialu_reg);
12299 %}
12300
12301 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
12302 %{
12303 predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12304 match(Set dst (RotateLeft src shift));
12305 format %{ "rolxl $dst, $src, $shift" %}
12306 ins_encode %{
12307 int shift = 32 - ($shift$$constant & 31);
12308 __ rorxl($dst$$Register, $src$$Register, shift);
12309 %}
12310 ins_pipe(ialu_reg_reg);
12311 %}
12312
12313 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
12314 %{
12315 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12316 match(Set dst (RotateLeft (LoadI src) shift));
12317 ins_cost(175);
12318 format %{ "rolxl $dst, $src, $shift" %}
12319 ins_encode %{
12320 int shift = 32 - ($shift$$constant & 31);
12321 __ rorxl($dst$$Register, $src$$Address, shift);
12322 %}
12323 ins_pipe(ialu_reg_mem);
12324 %}
12325
12326 // Rotate Left by variable
12327 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
12328 %{
12329 predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
12330 match(Set dst (RotateLeft dst shift));
12331 effect(KILL cr);
12332 format %{ "roll $dst, $shift" %}
12333 ins_encode %{
12334 __ roll($dst$$Register);
12335 %}
12336 ins_pipe(ialu_reg_reg);
12337 %}
12338
12339 // Rotate Left by variable
12340 instruct rolI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
12341 %{
12342 predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
12343 match(Set dst (RotateLeft src shift));
12344 effect(KILL cr);
12345 flag(PD::Flag_ndd_demotable_opr1);
12346
12347 format %{ "eroll $dst, $src, $shift\t# rotate left (int ndd)" %}
12348 ins_encode %{
12349 __ eroll($dst$$Register, $src$$Register, false);
12350 %}
12351 ins_pipe(ialu_reg_reg);
12352 %}
12353
12354 // Rotate Right by constant.
12355 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
12356 %{
12357 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12358 match(Set dst (RotateRight dst shift));
12359 effect(KILL cr);
12360 format %{ "rorl $dst, $shift" %}
12361 ins_encode %{
12362 __ rorl($dst$$Register, $shift$$constant);
12363 %}
12364 ins_pipe(ialu_reg);
12365 %}
12366
12367 // Rotate Right by constant.
12368 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
12369 %{
12370 predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12371 match(Set dst (RotateRight src shift));
12372 format %{ "rorxl $dst, $src, $shift" %}
12373 ins_encode %{
12374 __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
12375 %}
12376 ins_pipe(ialu_reg_reg);
12377 %}
12378
12379 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
12380 %{
12381 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
12382 match(Set dst (RotateRight (LoadI src) shift));
12383 ins_cost(175);
12384 format %{ "rorxl $dst, $src, $shift" %}
12385 ins_encode %{
12386 __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
12387 %}
12388 ins_pipe(ialu_reg_mem);
12389 %}
12390
12391 // Rotate Right by variable
12392 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
12393 %{
12394 predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
12395 match(Set dst (RotateRight dst shift));
12396 effect(KILL cr);
12397 format %{ "rorl $dst, $shift" %}
12398 ins_encode %{
12399 __ rorl($dst$$Register);
12400 %}
12401 ins_pipe(ialu_reg_reg);
12402 %}
12403
12404 // Rotate Right by variable
12405 instruct rorI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
12406 %{
12407 predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
12408 match(Set dst (RotateRight src shift));
12409 effect(KILL cr);
12410 flag(PD::Flag_ndd_demotable_opr1);
12411
12412 format %{ "erorl $dst, $src, $shift\t# rotate right(int ndd)" %}
12413 ins_encode %{
12414 __ erorl($dst$$Register, $src$$Register, false);
12415 %}
12416 ins_pipe(ialu_reg_reg);
12417 %}
12418
12419 // Rotate Left by constant.
12420 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
12421 %{
12422 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12423 match(Set dst (RotateLeft dst shift));
12424 effect(KILL cr);
12425 format %{ "rolq $dst, $shift" %}
12426 ins_encode %{
12427 __ rolq($dst$$Register, $shift$$constant);
12428 %}
12429 ins_pipe(ialu_reg);
12430 %}
12431
12432 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
12433 %{
12434 predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12435 match(Set dst (RotateLeft src shift));
12436 format %{ "rolxq $dst, $src, $shift" %}
12437 ins_encode %{
12438 int shift = 64 - ($shift$$constant & 63);
12439 __ rorxq($dst$$Register, $src$$Register, shift);
12440 %}
12441 ins_pipe(ialu_reg_reg);
12442 %}
12443
12444 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
12445 %{
12446 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12447 match(Set dst (RotateLeft (LoadL src) shift));
12448 ins_cost(175);
12449 format %{ "rolxq $dst, $src, $shift" %}
12450 ins_encode %{
12451 int shift = 64 - ($shift$$constant & 63);
12452 __ rorxq($dst$$Register, $src$$Address, shift);
12453 %}
12454 ins_pipe(ialu_reg_mem);
12455 %}
12456
12457 // Rotate Left by variable
12458 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12459 %{
12460 predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
12461 match(Set dst (RotateLeft dst shift));
12462 effect(KILL cr);
12463
12464 format %{ "rolq $dst, $shift" %}
12465 ins_encode %{
12466 __ rolq($dst$$Register);
12467 %}
12468 ins_pipe(ialu_reg_reg);
12469 %}
12470
12471 // Rotate Left by variable
12472 instruct rolL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
12473 %{
12474 predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
12475 match(Set dst (RotateLeft src shift));
12476 effect(KILL cr);
12477 flag(PD::Flag_ndd_demotable_opr1);
12478
12479 format %{ "erolq $dst, $src, $shift\t# rotate left(long ndd)" %}
12480 ins_encode %{
12481 __ erolq($dst$$Register, $src$$Register, false);
12482 %}
12483 ins_pipe(ialu_reg_reg);
12484 %}
12485
12486 // Rotate Right by constant.
12487 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
12488 %{
12489 predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12490 match(Set dst (RotateRight dst shift));
12491 effect(KILL cr);
12492 format %{ "rorq $dst, $shift" %}
12493 ins_encode %{
12494 __ rorq($dst$$Register, $shift$$constant);
12495 %}
12496 ins_pipe(ialu_reg);
12497 %}
12498
12499 // Rotate Right by constant
12500 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
12501 %{
12502 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12503 match(Set dst (RotateRight src shift));
12504 format %{ "rorxq $dst, $src, $shift" %}
12505 ins_encode %{
12506 __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
12507 %}
12508 ins_pipe(ialu_reg_reg);
12509 %}
12510
12511 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
12512 %{
12513 predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
12514 match(Set dst (RotateRight (LoadL src) shift));
12515 ins_cost(175);
12516 format %{ "rorxq $dst, $src, $shift" %}
12517 ins_encode %{
12518 __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
12519 %}
12520 ins_pipe(ialu_reg_mem);
12521 %}
12522
12523 // Rotate Right by variable
12524 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
12525 %{
12526 predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
12527 match(Set dst (RotateRight dst shift));
12528 effect(KILL cr);
12529 format %{ "rorq $dst, $shift" %}
12530 ins_encode %{
12531 __ rorq($dst$$Register);
12532 %}
12533 ins_pipe(ialu_reg_reg);
12534 %}
12535
12536 // Rotate Right by variable
12537 instruct rorL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
12538 %{
12539 predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
12540 match(Set dst (RotateRight src shift));
12541 effect(KILL cr);
12542 flag(PD::Flag_ndd_demotable_opr1);
12543
12544 format %{ "erorq $dst, $src, $shift\t# rotate right(long ndd)" %}
12545 ins_encode %{
12546 __ erorq($dst$$Register, $src$$Register, false);
12547 %}
12548 ins_pipe(ialu_reg_reg);
12549 %}
12550
12551 //----------------------------- CompressBits/ExpandBits ------------------------
12552
12553 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
12554 predicate(n->bottom_type()->isa_long());
12555 match(Set dst (CompressBits src mask));
12556 format %{ "pextq $dst, $src, $mask\t! parallel bit extract" %}
12557 ins_encode %{
12558 __ pextq($dst$$Register, $src$$Register, $mask$$Register);
12559 %}
12560 ins_pipe( pipe_slow );
12561 %}
12562
12563 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
12564 predicate(n->bottom_type()->isa_long());
12565 match(Set dst (ExpandBits src mask));
12566 format %{ "pdepq $dst, $src, $mask\t! parallel bit deposit" %}
12567 ins_encode %{
12568 __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
12569 %}
12570 ins_pipe( pipe_slow );
12571 %}
12572
12573 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
12574 predicate(n->bottom_type()->isa_long());
12575 match(Set dst (CompressBits src (LoadL mask)));
12576 format %{ "pextq $dst, $src, $mask\t! parallel bit extract" %}
12577 ins_encode %{
12578 __ pextq($dst$$Register, $src$$Register, $mask$$Address);
12579 %}
12580 ins_pipe( pipe_slow );
12581 %}
12582
12583 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
12584 predicate(n->bottom_type()->isa_long());
12585 match(Set dst (ExpandBits src (LoadL mask)));
12586 format %{ "pdepq $dst, $src, $mask\t! parallel bit deposit" %}
12587 ins_encode %{
12588 __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
12589 %}
12590 ins_pipe( pipe_slow );
12591 %}
12592
12593
12594 // Logical Instructions
12595
12596 // Integer Logical Instructions
12597
12598 // And Instructions
12599 // And Register with Register
12600 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
12601 %{
12602 predicate(!UseAPX);
12603 match(Set dst (AndI dst src));
12604 effect(KILL cr);
12605 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12606
12607 format %{ "andl $dst, $src\t# int" %}
12608 ins_encode %{
12609 __ andl($dst$$Register, $src$$Register);
12610 %}
12611 ins_pipe(ialu_reg_reg);
12612 %}
12613
12614 // And Register with Register using New Data Destination (NDD)
12615 instruct andI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
12616 %{
12617 predicate(UseAPX);
12618 match(Set dst (AndI src1 src2));
12619 effect(KILL cr);
12620 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
12621
12622 format %{ "eandl $dst, $src1, $src2\t# int ndd" %}
12623 ins_encode %{
12624 __ eandl($dst$$Register, $src1$$Register, $src2$$Register, false);
12625
12626 %}
12627 ins_pipe(ialu_reg_reg);
12628 %}
12629
12630 // And Register with Immediate 255
12631 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
12632 %{
12633 match(Set dst (AndI src mask));
12634
12635 format %{ "movzbl $dst, $src\t# int & 0xFF" %}
12636 ins_encode %{
12637 __ movzbl($dst$$Register, $src$$Register);
12638 %}
12639 ins_pipe(ialu_reg);
12640 %}
12641
12642 // And Register with Immediate 255 and promote to long
12643 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
12644 %{
12645 match(Set dst (ConvI2L (AndI src mask)));
12646
12647 format %{ "movzbl $dst, $src\t# int & 0xFF -> long" %}
12648 ins_encode %{
12649 __ movzbl($dst$$Register, $src$$Register);
12650 %}
12651 ins_pipe(ialu_reg);
12652 %}
12653
12654 // And Register with Immediate 65535
12655 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
12656 %{
12657 match(Set dst (AndI src mask));
12658
12659 format %{ "movzwl $dst, $src\t# int & 0xFFFF" %}
12660 ins_encode %{
12661 __ movzwl($dst$$Register, $src$$Register);
12662 %}
12663 ins_pipe(ialu_reg);
12664 %}
12665
12666 // And Register with Immediate 65535 and promote to long
12667 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
12668 %{
12669 match(Set dst (ConvI2L (AndI src mask)));
12670
12671 format %{ "movzwl $dst, $src\t# int & 0xFFFF -> long" %}
12672 ins_encode %{
12673 __ movzwl($dst$$Register, $src$$Register);
12674 %}
12675 ins_pipe(ialu_reg);
12676 %}
12677
12678 // Can skip int2long conversions after AND with small bitmask
12679 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src, immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
12680 %{
12681 predicate(VM_Version::supports_bmi2());
12682 ins_cost(125);
12683 effect(TEMP tmp, KILL cr);
12684 match(Set dst (ConvI2L (AndI src mask)));
12685 format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int & immI_Pow2M1 -> long" %}
12686 ins_encode %{
12687 __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
12688 __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
12689 %}
12690 ins_pipe(ialu_reg_reg);
12691 %}
12692
12693 // And Register with Immediate
12694 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
12695 %{
12696 predicate(!UseAPX);
12697 match(Set dst (AndI dst src));
12698 effect(KILL cr);
12699 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12700
12701 format %{ "andl $dst, $src\t# int" %}
12702 ins_encode %{
12703 __ andl($dst$$Register, $src$$constant);
12704 %}
12705 ins_pipe(ialu_reg);
12706 %}
12707
12708 instruct andI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
12709 %{
12710 predicate(UseAPX);
12711 match(Set dst (AndI src1 src2));
12712 effect(KILL cr);
12713 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
12714
12715 format %{ "eandl $dst, $src1, $src2\t# int ndd" %}
12716 ins_encode %{
12717 __ eandl($dst$$Register, $src1$$Register, $src2$$constant, false);
12718 %}
12719 ins_pipe(ialu_reg);
12720 %}
12721
12722 // And Register with Memory
12723 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
12724 %{
12725 match(Set dst (AndI dst (LoadI src)));
12726 effect(KILL cr);
12727 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12728
12729 ins_cost(150);
12730 format %{ "andl $dst, $src\t# int" %}
12731 ins_encode %{
12732 __ andl($dst$$Register, $src$$Address);
12733 %}
12734 ins_pipe(ialu_reg_mem);
12735 %}
12736
12737 // And Memory with Register
12738 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
12739 %{
12740 match(Set dst (StoreB dst (AndI (LoadB dst) src)));
12741 effect(KILL cr);
12742 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12743
12744 ins_cost(150);
12745 format %{ "andb $dst, $src\t# byte" %}
12746 ins_encode %{
12747 __ andb($dst$$Address, $src$$Register);
12748 %}
12749 ins_pipe(ialu_mem_reg);
12750 %}
12751
12752 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
12753 %{
12754 match(Set dst (StoreI dst (AndI (LoadI dst) src)));
12755 effect(KILL cr);
12756 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12757
12758 ins_cost(150);
12759 format %{ "andl $dst, $src\t# int" %}
12760 ins_encode %{
12761 __ andl($dst$$Address, $src$$Register);
12762 %}
12763 ins_pipe(ialu_mem_reg);
12764 %}
12765
12766 // And Memory with Immediate
12767 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
12768 %{
12769 match(Set dst (StoreI dst (AndI (LoadI dst) src)));
12770 effect(KILL cr);
12771 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12772
12773 ins_cost(125);
12774 format %{ "andl $dst, $src\t# int" %}
12775 ins_encode %{
12776 __ andl($dst$$Address, $src$$constant);
12777 %}
12778 ins_pipe(ialu_mem_imm);
12779 %}
12780
12781 // BMI1 instructions
12782 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
12783 match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
12784 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12785 effect(KILL cr);
12786 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12787
12788 ins_cost(125);
12789 format %{ "andnl $dst, $src1, $src2" %}
12790
12791 ins_encode %{
12792 __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
12793 %}
12794 ins_pipe(ialu_reg_mem);
12795 %}
12796
12797 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
12798 match(Set dst (AndI (XorI src1 minus_1) src2));
12799 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12800 effect(KILL cr);
12801 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12802
12803 format %{ "andnl $dst, $src1, $src2" %}
12804
12805 ins_encode %{
12806 __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
12807 %}
12808 ins_pipe(ialu_reg);
12809 %}
12810
12811 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
12812 match(Set dst (AndI (SubI imm_zero src) src));
12813 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12814 effect(KILL cr);
12815 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12816
12817 format %{ "blsil $dst, $src" %}
12818
12819 ins_encode %{
12820 __ blsil($dst$$Register, $src$$Register);
12821 %}
12822 ins_pipe(ialu_reg);
12823 %}
12824
12825 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
12826 match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
12827 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12828 effect(KILL cr);
12829 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12830
12831 ins_cost(125);
12832 format %{ "blsil $dst, $src" %}
12833
12834 ins_encode %{
12835 __ blsil($dst$$Register, $src$$Address);
12836 %}
12837 ins_pipe(ialu_reg_mem);
12838 %}
12839
12840 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
12841 %{
12842 match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
12843 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12844 effect(KILL cr);
12845 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
12846
12847 ins_cost(125);
12848 format %{ "blsmskl $dst, $src" %}
12849
12850 ins_encode %{
12851 __ blsmskl($dst$$Register, $src$$Address);
12852 %}
12853 ins_pipe(ialu_reg_mem);
12854 %}
12855
12856 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
12857 %{
12858 match(Set dst (XorI (AddI src minus_1) src));
12859 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12860 effect(KILL cr);
12861 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
12862
12863 format %{ "blsmskl $dst, $src" %}
12864
12865 ins_encode %{
12866 __ blsmskl($dst$$Register, $src$$Register);
12867 %}
12868
12869 ins_pipe(ialu_reg);
12870 %}
12871
12872 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
12873 %{
12874 match(Set dst (AndI (AddI src minus_1) src) );
12875 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12876 effect(KILL cr);
12877 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12878
12879 format %{ "blsrl $dst, $src" %}
12880
12881 ins_encode %{
12882 __ blsrl($dst$$Register, $src$$Register);
12883 %}
12884
12885 ins_pipe(ialu_reg_mem);
12886 %}
12887
12888 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
12889 %{
12890 match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
12891 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
12892 effect(KILL cr);
12893 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
12894
12895 ins_cost(125);
12896 format %{ "blsrl $dst, $src" %}
12897
12898 ins_encode %{
12899 __ blsrl($dst$$Register, $src$$Address);
12900 %}
12901
12902 ins_pipe(ialu_reg);
12903 %}
12904
12905 // Or Instructions
12906 // Or Register with Register
12907 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
12908 %{
12909 predicate(!UseAPX);
12910 match(Set dst (OrI dst src));
12911 effect(KILL cr);
12912 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12913
12914 format %{ "orl $dst, $src\t# int" %}
12915 ins_encode %{
12916 __ orl($dst$$Register, $src$$Register);
12917 %}
12918 ins_pipe(ialu_reg_reg);
12919 %}
12920
12921 // Or Register with Register using New Data Destination (NDD)
12922 instruct orI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
12923 %{
12924 predicate(UseAPX);
12925 match(Set dst (OrI src1 src2));
12926 effect(KILL cr);
12927 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
12928
12929 format %{ "eorl $dst, $src1, $src2\t# int ndd" %}
12930 ins_encode %{
12931 __ eorl($dst$$Register, $src1$$Register, $src2$$Register, false);
12932 %}
12933 ins_pipe(ialu_reg_reg);
12934 %}
12935
12936 // Or Register with Immediate
12937 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
12938 %{
12939 predicate(!UseAPX);
12940 match(Set dst (OrI dst src));
12941 effect(KILL cr);
12942 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12943
12944 format %{ "orl $dst, $src\t# int" %}
12945 ins_encode %{
12946 __ orl($dst$$Register, $src$$constant);
12947 %}
12948 ins_pipe(ialu_reg);
12949 %}
12950
12951 instruct orI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
12952 %{
12953 predicate(UseAPX);
12954 match(Set dst (OrI src1 src2));
12955 effect(KILL cr);
12956 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
12957
12958 format %{ "eorl $dst, $src1, $src2\t# int ndd" %}
12959 ins_encode %{
12960 __ eorl($dst$$Register, $src1$$Register, $src2$$constant, false);
12961 %}
12962 ins_pipe(ialu_reg);
12963 %}
12964
12965 instruct orI_rReg_imm_rReg_ndd(rRegI dst, immI src1, rRegI src2, rFlagsReg cr)
12966 %{
12967 predicate(UseAPX);
12968 match(Set dst (OrI src1 src2));
12969 effect(KILL cr);
12970 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
12971
12972 format %{ "eorl $dst, $src2, $src1\t# int ndd" %}
12973 ins_encode %{
12974 __ eorl($dst$$Register, $src2$$Register, $src1$$constant, false);
12975 %}
12976 ins_pipe(ialu_reg);
12977 %}
12978
12979 // Or Register with Memory
12980 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
12981 %{
12982 match(Set dst (OrI dst (LoadI src)));
12983 effect(KILL cr);
12984 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
12985
12986 ins_cost(150);
12987 format %{ "orl $dst, $src\t# int" %}
12988 ins_encode %{
12989 __ orl($dst$$Register, $src$$Address);
12990 %}
12991 ins_pipe(ialu_reg_mem);
12992 %}
12993
12994 // Or Memory with Register
12995 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
12996 %{
12997 match(Set dst (StoreB dst (OrI (LoadB dst) src)));
12998 effect(KILL cr);
12999 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13000
13001 ins_cost(150);
13002 format %{ "orb $dst, $src\t# byte" %}
13003 ins_encode %{
13004 __ orb($dst$$Address, $src$$Register);
13005 %}
13006 ins_pipe(ialu_mem_reg);
13007 %}
13008
13009 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
13010 %{
13011 match(Set dst (StoreI dst (OrI (LoadI dst) src)));
13012 effect(KILL cr);
13013 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13014
13015 ins_cost(150);
13016 format %{ "orl $dst, $src\t# int" %}
13017 ins_encode %{
13018 __ orl($dst$$Address, $src$$Register);
13019 %}
13020 ins_pipe(ialu_mem_reg);
13021 %}
13022
13023 // Or Memory with Immediate
13024 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
13025 %{
13026 match(Set dst (StoreI dst (OrI (LoadI dst) src)));
13027 effect(KILL cr);
13028 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13029
13030 ins_cost(125);
13031 format %{ "orl $dst, $src\t# int" %}
13032 ins_encode %{
13033 __ orl($dst$$Address, $src$$constant);
13034 %}
13035 ins_pipe(ialu_mem_imm);
13036 %}
13037
13038 // Xor Instructions
13039 // Xor Register with Register
13040 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
13041 %{
13042 predicate(!UseAPX);
13043 match(Set dst (XorI dst src));
13044 effect(KILL cr);
13045 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13046
13047 format %{ "xorl $dst, $src\t# int" %}
13048 ins_encode %{
13049 __ xorl($dst$$Register, $src$$Register);
13050 %}
13051 ins_pipe(ialu_reg_reg);
13052 %}
13053
13054 // Xor Register with Register using New Data Destination (NDD)
13055 instruct xorI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13056 %{
13057 predicate(UseAPX);
13058 match(Set dst (XorI src1 src2));
13059 effect(KILL cr);
13060 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13061
13062 format %{ "exorl $dst, $src1, $src2\t# int ndd" %}
13063 ins_encode %{
13064 __ exorl($dst$$Register, $src1$$Register, $src2$$Register, false);
13065 %}
13066 ins_pipe(ialu_reg_reg);
13067 %}
13068
13069 // Xor Register with Immediate -1
13070 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm)
13071 %{
13072 predicate(!UseAPX);
13073 match(Set dst (XorI dst imm));
13074
13075 format %{ "notl $dst" %}
13076 ins_encode %{
13077 __ notl($dst$$Register);
13078 %}
13079 ins_pipe(ialu_reg);
13080 %}
13081
13082 instruct xorI_rReg_im1_ndd(rRegI dst, rRegI src, immI_M1 imm)
13083 %{
13084 match(Set dst (XorI src imm));
13085 predicate(UseAPX);
13086 flag(PD::Flag_ndd_demotable_opr1);
13087
13088 format %{ "enotl $dst, $src" %}
13089 ins_encode %{
13090 __ enotl($dst$$Register, $src$$Register);
13091 %}
13092 ins_pipe(ialu_reg);
13093 %}
13094
13095 // Xor Register with Immediate
13096 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
13097 %{
13098 // Strict predicate check to make selection of xorI_rReg_im1 cost agnostic if immI src is -1.
13099 predicate(!UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
13100 match(Set dst (XorI dst src));
13101 effect(KILL cr);
13102 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13103
13104 format %{ "xorl $dst, $src\t# int" %}
13105 ins_encode %{
13106 __ xorl($dst$$Register, $src$$constant);
13107 %}
13108 ins_pipe(ialu_reg);
13109 %}
13110
13111 instruct xorI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
13112 %{
13113 // Strict predicate check to make selection of xorI_rReg_im1_ndd cost agnostic if immI src2 is -1.
13114 predicate(UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
13115 match(Set dst (XorI src1 src2));
13116 effect(KILL cr);
13117 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13118
13119 format %{ "exorl $dst, $src1, $src2\t# int ndd" %}
13120 ins_encode %{
13121 __ exorl($dst$$Register, $src1$$Register, $src2$$constant, false);
13122 %}
13123 ins_pipe(ialu_reg);
13124 %}
13125
13126 // Xor Register with Memory
13127 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
13128 %{
13129 match(Set dst (XorI dst (LoadI src)));
13130 effect(KILL cr);
13131 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13132
13133 ins_cost(150);
13134 format %{ "xorl $dst, $src\t# int" %}
13135 ins_encode %{
13136 __ xorl($dst$$Register, $src$$Address);
13137 %}
13138 ins_pipe(ialu_reg_mem);
13139 %}
13140
13141 // Xor Memory with Register
13142 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
13143 %{
13144 match(Set dst (StoreB dst (XorI (LoadB dst) src)));
13145 effect(KILL cr);
13146 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13147
13148 ins_cost(150);
13149 format %{ "xorb $dst, $src\t# byte" %}
13150 ins_encode %{
13151 __ xorb($dst$$Address, $src$$Register);
13152 %}
13153 ins_pipe(ialu_mem_reg);
13154 %}
13155
13156 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
13157 %{
13158 match(Set dst (StoreI dst (XorI (LoadI dst) src)));
13159 effect(KILL cr);
13160 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13161
13162 ins_cost(150);
13163 format %{ "xorl $dst, $src\t# int" %}
13164 ins_encode %{
13165 __ xorl($dst$$Address, $src$$Register);
13166 %}
13167 ins_pipe(ialu_mem_reg);
13168 %}
13169
13170 // Xor Memory with Immediate
13171 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
13172 %{
13173 match(Set dst (StoreI dst (XorI (LoadI dst) src)));
13174 effect(KILL cr);
13175 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13176
13177 ins_cost(125);
13178 format %{ "xorl $dst, $src\t# int" %}
13179 ins_encode %{
13180 __ xorl($dst$$Address, $src$$constant);
13181 %}
13182 ins_pipe(ialu_mem_imm);
13183 %}
13184
13185
13186 // Long Logical Instructions
13187
13188 // And Instructions
13189 // And Register with Register
13190 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
13191 %{
13192 predicate(!UseAPX);
13193 match(Set dst (AndL dst src));
13194 effect(KILL cr);
13195 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13196
13197 format %{ "andq $dst, $src\t# long" %}
13198 ins_encode %{
13199 __ andq($dst$$Register, $src$$Register);
13200 %}
13201 ins_pipe(ialu_reg_reg);
13202 %}
13203
13204 // And Register with Register using New Data Destination (NDD)
13205 instruct andL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
13206 %{
13207 predicate(UseAPX);
13208 match(Set dst (AndL src1 src2));
13209 effect(KILL cr);
13210 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13211
13212 format %{ "eandq $dst, $src1, $src2\t# long ndd" %}
13213 ins_encode %{
13214 __ eandq($dst$$Register, $src1$$Register, $src2$$Register, false);
13215
13216 %}
13217 ins_pipe(ialu_reg_reg);
13218 %}
13219
13220 // And Register with Immediate 255
13221 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
13222 %{
13223 match(Set dst (AndL src mask));
13224
13225 format %{ "movzbl $dst, $src\t# long & 0xFF" %}
13226 ins_encode %{
13227 // movzbl zeroes out the upper 32-bit and does not need REX.W
13228 __ movzbl($dst$$Register, $src$$Register);
13229 %}
13230 ins_pipe(ialu_reg);
13231 %}
13232
13233 // And Register with Immediate 65535
13234 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
13235 %{
13236 match(Set dst (AndL src mask));
13237
13238 format %{ "movzwl $dst, $src\t# long & 0xFFFF" %}
13239 ins_encode %{
13240 // movzwl zeroes out the upper 32-bit and does not need REX.W
13241 __ movzwl($dst$$Register, $src$$Register);
13242 %}
13243 ins_pipe(ialu_reg);
13244 %}
13245
13246 // And Register with Immediate
13247 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
13248 %{
13249 predicate(!UseAPX);
13250 match(Set dst (AndL dst src));
13251 effect(KILL cr);
13252 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13253
13254 format %{ "andq $dst, $src\t# long" %}
13255 ins_encode %{
13256 __ andq($dst$$Register, $src$$constant);
13257 %}
13258 ins_pipe(ialu_reg);
13259 %}
13260
13261 instruct andL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
13262 %{
13263 predicate(UseAPX);
13264 match(Set dst (AndL src1 src2));
13265 effect(KILL cr);
13266 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13267
13268 format %{ "eandq $dst, $src1, $src2\t# long ndd" %}
13269 ins_encode %{
13270 __ eandq($dst$$Register, $src1$$Register, $src2$$constant, false);
13271 %}
13272 ins_pipe(ialu_reg);
13273 %}
13274
13275 // And Register with Memory
13276 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
13277 %{
13278 match(Set dst (AndL dst (LoadL src)));
13279 effect(KILL cr);
13280 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13281
13282 ins_cost(150);
13283 format %{ "andq $dst, $src\t# long" %}
13284 ins_encode %{
13285 __ andq($dst$$Register, $src$$Address);
13286 %}
13287 ins_pipe(ialu_reg_mem);
13288 %}
13289
13290 // And Memory with Register
13291 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
13292 %{
13293 match(Set dst (StoreL dst (AndL (LoadL dst) src)));
13294 effect(KILL cr);
13295 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13296
13297 ins_cost(150);
13298 format %{ "andq $dst, $src\t# long" %}
13299 ins_encode %{
13300 __ andq($dst$$Address, $src$$Register);
13301 %}
13302 ins_pipe(ialu_mem_reg);
13303 %}
13304
13305 // And Memory with Immediate
13306 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
13307 %{
13308 match(Set dst (StoreL dst (AndL (LoadL dst) src)));
13309 effect(KILL cr);
13310 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13311
13312 ins_cost(125);
13313 format %{ "andq $dst, $src\t# long" %}
13314 ins_encode %{
13315 __ andq($dst$$Address, $src$$constant);
13316 %}
13317 ins_pipe(ialu_mem_imm);
13318 %}
13319
13320 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
13321 %{
13322 // con should be a pure 64-bit immediate given that not(con) is a power of 2
13323 // because AND/OR works well enough for 8/32-bit values.
13324 predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
13325
13326 match(Set dst (StoreL dst (AndL (LoadL dst) con)));
13327 effect(KILL cr);
13328
13329 ins_cost(125);
13330 format %{ "btrq $dst, log2(not($con))\t# long" %}
13331 ins_encode %{
13332 __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
13333 %}
13334 ins_pipe(ialu_mem_imm);
13335 %}
13336
13337 // BMI1 instructions
13338 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
13339 match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
13340 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13341 effect(KILL cr);
13342 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13343
13344 ins_cost(125);
13345 format %{ "andnq $dst, $src1, $src2" %}
13346
13347 ins_encode %{
13348 __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
13349 %}
13350 ins_pipe(ialu_reg_mem);
13351 %}
13352
13353 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
13354 match(Set dst (AndL (XorL src1 minus_1) src2));
13355 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13356 effect(KILL cr);
13357 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13358
13359 format %{ "andnq $dst, $src1, $src2" %}
13360
13361 ins_encode %{
13362 __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
13363 %}
13364 ins_pipe(ialu_reg_mem);
13365 %}
13366
13367 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
13368 match(Set dst (AndL (SubL imm_zero src) src));
13369 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13370 effect(KILL cr);
13371 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13372
13373 format %{ "blsiq $dst, $src" %}
13374
13375 ins_encode %{
13376 __ blsiq($dst$$Register, $src$$Register);
13377 %}
13378 ins_pipe(ialu_reg);
13379 %}
13380
13381 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
13382 match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
13383 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13384 effect(KILL cr);
13385 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13386
13387 ins_cost(125);
13388 format %{ "blsiq $dst, $src" %}
13389
13390 ins_encode %{
13391 __ blsiq($dst$$Register, $src$$Address);
13392 %}
13393 ins_pipe(ialu_reg_mem);
13394 %}
13395
13396 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
13397 %{
13398 match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
13399 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13400 effect(KILL cr);
13401 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
13402
13403 ins_cost(125);
13404 format %{ "blsmskq $dst, $src" %}
13405
13406 ins_encode %{
13407 __ blsmskq($dst$$Register, $src$$Address);
13408 %}
13409 ins_pipe(ialu_reg_mem);
13410 %}
13411
13412 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
13413 %{
13414 match(Set dst (XorL (AddL src minus_1) src));
13415 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13416 effect(KILL cr);
13417 flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
13418
13419 format %{ "blsmskq $dst, $src" %}
13420
13421 ins_encode %{
13422 __ blsmskq($dst$$Register, $src$$Register);
13423 %}
13424
13425 ins_pipe(ialu_reg);
13426 %}
13427
13428 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
13429 %{
13430 match(Set dst (AndL (AddL src minus_1) src) );
13431 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13432 effect(KILL cr);
13433 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13434
13435 format %{ "blsrq $dst, $src" %}
13436
13437 ins_encode %{
13438 __ blsrq($dst$$Register, $src$$Register);
13439 %}
13440
13441 ins_pipe(ialu_reg);
13442 %}
13443
13444 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
13445 %{
13446 match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
13447 predicate(VM_Version::supports_bmi1() && VM_Version::supports_avx());
13448 effect(KILL cr);
13449 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
13450
13451 ins_cost(125);
13452 format %{ "blsrq $dst, $src" %}
13453
13454 ins_encode %{
13455 __ blsrq($dst$$Register, $src$$Address);
13456 %}
13457
13458 ins_pipe(ialu_reg);
13459 %}
13460
13461 // Or Instructions
13462 // Or Register with Register
13463 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
13464 %{
13465 predicate(!UseAPX);
13466 match(Set dst (OrL dst src));
13467 effect(KILL cr);
13468 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13469
13470 format %{ "orq $dst, $src\t# long" %}
13471 ins_encode %{
13472 __ orq($dst$$Register, $src$$Register);
13473 %}
13474 ins_pipe(ialu_reg_reg);
13475 %}
13476
13477 // Or Register with Register using New Data Destination (NDD)
13478 instruct orL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
13479 %{
13480 predicate(UseAPX);
13481 match(Set dst (OrL src1 src2));
13482 effect(KILL cr);
13483 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13484
13485 format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
13486 ins_encode %{
13487 __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
13488
13489 %}
13490 ins_pipe(ialu_reg_reg);
13491 %}
13492
13493 // Use any_RegP to match R15 (TLS register) without spilling.
13494 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
13495 predicate(!UseAPX);
13496 match(Set dst (OrL dst (CastP2X src)));
13497 effect(KILL cr);
13498 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13499
13500 format %{ "orq $dst, $src\t# long" %}
13501 ins_encode %{
13502 __ orq($dst$$Register, $src$$Register);
13503 %}
13504 ins_pipe(ialu_reg_reg);
13505 %}
13506
13507 instruct orL_rReg_castP2X_ndd(rRegL dst, any_RegP src1, any_RegP src2, rFlagsReg cr) %{
13508 predicate(UseAPX);
13509 match(Set dst (OrL src1 (CastP2X src2)));
13510 effect(KILL cr);
13511 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13512
13513 format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
13514 ins_encode %{
13515 __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
13516 %}
13517 ins_pipe(ialu_reg_reg);
13518 %}
13519
13520 // Or Register with Immediate
13521 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
13522 %{
13523 predicate(!UseAPX);
13524 match(Set dst (OrL dst src));
13525 effect(KILL cr);
13526 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13527
13528 format %{ "orq $dst, $src\t# long" %}
13529 ins_encode %{
13530 __ orq($dst$$Register, $src$$constant);
13531 %}
13532 ins_pipe(ialu_reg);
13533 %}
13534
13535 instruct orL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
13536 %{
13537 predicate(UseAPX);
13538 match(Set dst (OrL src1 src2));
13539 effect(KILL cr);
13540 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13541
13542 format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
13543 ins_encode %{
13544 __ eorq($dst$$Register, $src1$$Register, $src2$$constant, false);
13545 %}
13546 ins_pipe(ialu_reg);
13547 %}
13548
13549 instruct orL_rReg_imm_rReg_ndd(rRegL dst, immL32 src1, rRegL src2, rFlagsReg cr)
13550 %{
13551 predicate(UseAPX);
13552 match(Set dst (OrL src1 src2));
13553 effect(KILL cr);
13554 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13555
13556 format %{ "eorq $dst, $src2, $src1\t# long ndd" %}
13557 ins_encode %{
13558 __ eorq($dst$$Register, $src2$$Register, $src1$$constant, false);
13559 %}
13560 ins_pipe(ialu_reg);
13561 %}
13562
13563 // Or Register with Memory
13564 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
13565 %{
13566 match(Set dst (OrL dst (LoadL src)));
13567 effect(KILL cr);
13568 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13569
13570 ins_cost(150);
13571 format %{ "orq $dst, $src\t# long" %}
13572 ins_encode %{
13573 __ orq($dst$$Register, $src$$Address);
13574 %}
13575 ins_pipe(ialu_reg_mem);
13576 %}
13577
13578 // Or Memory with Register
13579 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
13580 %{
13581 match(Set dst (StoreL dst (OrL (LoadL dst) src)));
13582 effect(KILL cr);
13583 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13584
13585 ins_cost(150);
13586 format %{ "orq $dst, $src\t# long" %}
13587 ins_encode %{
13588 __ orq($dst$$Address, $src$$Register);
13589 %}
13590 ins_pipe(ialu_mem_reg);
13591 %}
13592
13593 // Or Memory with Immediate
13594 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
13595 %{
13596 match(Set dst (StoreL dst (OrL (LoadL dst) src)));
13597 effect(KILL cr);
13598 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13599
13600 ins_cost(125);
13601 format %{ "orq $dst, $src\t# long" %}
13602 ins_encode %{
13603 __ orq($dst$$Address, $src$$constant);
13604 %}
13605 ins_pipe(ialu_mem_imm);
13606 %}
13607
13608 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
13609 %{
13610 // con should be a pure 64-bit power of 2 immediate
13611 // because AND/OR works well enough for 8/32-bit values.
13612 predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
13613
13614 match(Set dst (StoreL dst (OrL (LoadL dst) con)));
13615 effect(KILL cr);
13616
13617 ins_cost(125);
13618 format %{ "btsq $dst, log2($con)\t# long" %}
13619 ins_encode %{
13620 __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
13621 %}
13622 ins_pipe(ialu_mem_imm);
13623 %}
13624
13625 // Xor Instructions
13626 // Xor Register with Register
13627 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
13628 %{
13629 predicate(!UseAPX);
13630 match(Set dst (XorL dst src));
13631 effect(KILL cr);
13632 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13633
13634 format %{ "xorq $dst, $src\t# long" %}
13635 ins_encode %{
13636 __ xorq($dst$$Register, $src$$Register);
13637 %}
13638 ins_pipe(ialu_reg_reg);
13639 %}
13640
13641 // Xor Register with Register using New Data Destination (NDD)
13642 instruct xorL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
13643 %{
13644 predicate(UseAPX);
13645 match(Set dst (XorL src1 src2));
13646 effect(KILL cr);
13647 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1, PD::Flag_ndd_demotable_opr2);
13648
13649 format %{ "exorq $dst, $src1, $src2\t# long ndd" %}
13650 ins_encode %{
13651 __ exorq($dst$$Register, $src1$$Register, $src2$$Register, false);
13652 %}
13653 ins_pipe(ialu_reg_reg);
13654 %}
13655
13656 // Xor Register with Immediate -1
13657 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm)
13658 %{
13659 predicate(!UseAPX);
13660 match(Set dst (XorL dst imm));
13661
13662 format %{ "notq $dst" %}
13663 ins_encode %{
13664 __ notq($dst$$Register);
13665 %}
13666 ins_pipe(ialu_reg);
13667 %}
13668
13669 instruct xorL_rReg_im1_ndd(rRegL dst,rRegL src, immL_M1 imm)
13670 %{
13671 predicate(UseAPX);
13672 match(Set dst (XorL src imm));
13673 flag(PD::Flag_ndd_demotable_opr1);
13674
13675 format %{ "enotq $dst, $src" %}
13676 ins_encode %{
13677 __ enotq($dst$$Register, $src$$Register);
13678 %}
13679 ins_pipe(ialu_reg);
13680 %}
13681
13682 // Xor Register with Immediate
13683 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
13684 %{
13685 // Strict predicate check to make selection of xorL_rReg_im1 cost agnostic if immL32 src is -1.
13686 predicate(!UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
13687 match(Set dst (XorL dst src));
13688 effect(KILL cr);
13689 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13690
13691 format %{ "xorq $dst, $src\t# long" %}
13692 ins_encode %{
13693 __ xorq($dst$$Register, $src$$constant);
13694 %}
13695 ins_pipe(ialu_reg);
13696 %}
13697
13698 instruct xorL_rReg_rReg_imm(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
13699 %{
13700 // Strict predicate check to make selection of xorL_rReg_im1_ndd cost agnostic if immL32 src2 is -1.
13701 predicate(UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
13702 match(Set dst (XorL src1 src2));
13703 effect(KILL cr);
13704 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag, PD::Flag_ndd_demotable_opr1);
13705
13706 format %{ "exorq $dst, $src1, $src2\t# long ndd" %}
13707 ins_encode %{
13708 __ exorq($dst$$Register, $src1$$Register, $src2$$constant, false);
13709 %}
13710 ins_pipe(ialu_reg);
13711 %}
13712
13713 // Xor Register with Memory
13714 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
13715 %{
13716 match(Set dst (XorL dst (LoadL src)));
13717 effect(KILL cr);
13718 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13719
13720 ins_cost(150);
13721 format %{ "xorq $dst, $src\t# long" %}
13722 ins_encode %{
13723 __ xorq($dst$$Register, $src$$Address);
13724 %}
13725 ins_pipe(ialu_reg_mem);
13726 %}
13727
13728 // Xor Memory with Register
13729 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
13730 %{
13731 match(Set dst (StoreL dst (XorL (LoadL dst) src)));
13732 effect(KILL cr);
13733 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13734
13735 ins_cost(150);
13736 format %{ "xorq $dst, $src\t# long" %}
13737 ins_encode %{
13738 __ xorq($dst$$Address, $src$$Register);
13739 %}
13740 ins_pipe(ialu_mem_reg);
13741 %}
13742
13743 // Xor Memory with Immediate
13744 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
13745 %{
13746 match(Set dst (StoreL dst (XorL (LoadL dst) src)));
13747 effect(KILL cr);
13748 flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
13749
13750 ins_cost(125);
13751 format %{ "xorq $dst, $src\t# long" %}
13752 ins_encode %{
13753 __ xorq($dst$$Address, $src$$constant);
13754 %}
13755 ins_pipe(ialu_mem_imm);
13756 %}
13757
13758 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
13759 %{
13760 match(Set dst (CmpLTMask p q));
13761 effect(KILL cr);
13762
13763 ins_cost(400);
13764 format %{ "cmpl $p, $q\t# cmpLTMask\n\t"
13765 "setcc $dst \t# emits setlt + movzbl or setzul for APX"
13766 "negl $dst" %}
13767 ins_encode %{
13768 __ cmpl($p$$Register, $q$$Register);
13769 __ setcc(Assembler::less, $dst$$Register);
13770 __ negl($dst$$Register);
13771 %}
13772 ins_pipe(pipe_slow);
13773 %}
13774
13775 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
13776 %{
13777 match(Set dst (CmpLTMask dst zero));
13778 effect(KILL cr);
13779
13780 ins_cost(100);
13781 format %{ "sarl $dst, #31\t# cmpLTMask0" %}
13782 ins_encode %{
13783 __ sarl($dst$$Register, 31);
13784 %}
13785 ins_pipe(ialu_reg);
13786 %}
13787
13788 /* Better to save a register than avoid a branch */
13789 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
13790 %{
13791 match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
13792 effect(KILL cr);
13793 ins_cost(300);
13794 format %{ "subl $p,$q\t# cadd_cmpLTMask\n\t"
13795 "jge done\n\t"
13796 "addl $p,$y\n"
13797 "done: " %}
13798 ins_encode %{
13799 Register Rp = $p$$Register;
13800 Register Rq = $q$$Register;
13801 Register Ry = $y$$Register;
13802 Label done;
13803 __ subl(Rp, Rq);
13804 __ jccb(Assembler::greaterEqual, done);
13805 __ addl(Rp, Ry);
13806 __ bind(done);
13807 %}
13808 ins_pipe(pipe_cmplt);
13809 %}
13810
13811 /* Better to save a register than avoid a branch */
13812 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
13813 %{
13814 match(Set y (AndI (CmpLTMask p q) y));
13815 effect(KILL cr);
13816
13817 ins_cost(300);
13818
13819 format %{ "cmpl $p, $q\t# and_cmpLTMask\n\t"
13820 "jlt done\n\t"
13821 "xorl $y, $y\n"
13822 "done: " %}
13823 ins_encode %{
13824 Register Rp = $p$$Register;
13825 Register Rq = $q$$Register;
13826 Register Ry = $y$$Register;
13827 Label done;
13828 __ cmpl(Rp, Rq);
13829 __ jccb(Assembler::less, done);
13830 __ xorl(Ry, Ry);
13831 __ bind(done);
13832 %}
13833 ins_pipe(pipe_cmplt);
13834 %}
13835
13836
13837 //---------- FP Instructions------------------------------------------------
13838
13839 // Really expensive, avoid
13840 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
13841 %{
13842 match(Set cr (CmpF src1 src2));
13843
13844 ins_cost(500);
13845 format %{ "ucomiss $src1, $src2\n\t"
13846 "jnp,s exit\n\t"
13847 "pushfq\t# saw NaN, set CF\n\t"
13848 "andq [rsp], #0xffffff2b\n\t"
13849 "popfq\n"
13850 "exit:" %}
13851 ins_encode %{
13852 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
13853 emit_cmpfp_fixup(masm);
13854 %}
13855 ins_pipe(pipe_slow);
13856 %}
13857
13858 instruct cmpF_cc_regCF(rFlagsRegUCF cr, regF src1, regF src2) %{
13859 match(Set cr (CmpF src1 src2));
13860
13861 ins_cost(100);
13862 format %{ "ucomiss $src1, $src2" %}
13863 ins_encode %{
13864 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
13865 %}
13866 ins_pipe(pipe_slow);
13867 %}
13868
13869 instruct cmpF_cc_regCFE(rFlagsRegUCFE cr, regF src1, regF src2) %{
13870 match(Set cr (CmpF src1 src2));
13871
13872 ins_cost(100);
13873 format %{ "evucomxss $src1, $src2" %}
13874 ins_encode %{
13875 __ evucomxss($src1$$XMMRegister, $src2$$XMMRegister);
13876 %}
13877 ins_pipe(pipe_slow);
13878 %}
13879
13880 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
13881 match(Set cr (CmpF src1 (LoadF src2)));
13882
13883 ins_cost(100);
13884 format %{ "ucomiss $src1, $src2" %}
13885 ins_encode %{
13886 __ ucomiss($src1$$XMMRegister, $src2$$Address);
13887 %}
13888 ins_pipe(pipe_slow);
13889 %}
13890
13891 instruct cmpF_cc_memCFE(rFlagsRegUCFE cr, regF src1, memory src2) %{
13892 match(Set cr (CmpF src1 (LoadF src2)));
13893
13894 ins_cost(100);
13895 format %{ "evucomxss $src1, $src2" %}
13896 ins_encode %{
13897 __ evucomxss($src1$$XMMRegister, $src2$$Address);
13898 %}
13899 ins_pipe(pipe_slow);
13900 %}
13901
13902 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
13903 match(Set cr (CmpF src con));
13904
13905 ins_cost(100);
13906 format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
13907 ins_encode %{
13908 __ ucomiss($src$$XMMRegister, $constantaddress($con));
13909 %}
13910 ins_pipe(pipe_slow);
13911 %}
13912
13913 instruct cmpF_cc_immCFE(rFlagsRegUCFE cr, regF src, immF con) %{
13914 match(Set cr (CmpF src con));
13915
13916 ins_cost(100);
13917 format %{ "evucomxss $src, [$constantaddress]\t# load from constant table: float=$con" %}
13918 ins_encode %{
13919 __ evucomxss($src$$XMMRegister, $constantaddress($con));
13920 %}
13921 ins_pipe(pipe_slow);
13922 %}
13923
13924 // Really expensive, avoid
13925 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
13926 %{
13927 match(Set cr (CmpD src1 src2));
13928
13929 ins_cost(500);
13930 format %{ "ucomisd $src1, $src2\n\t"
13931 "jnp,s exit\n\t"
13932 "pushfq\t# saw NaN, set CF\n\t"
13933 "andq [rsp], #0xffffff2b\n\t"
13934 "popfq\n"
13935 "exit:" %}
13936 ins_encode %{
13937 __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
13938 emit_cmpfp_fixup(masm);
13939 %}
13940 ins_pipe(pipe_slow);
13941 %}
13942
13943 instruct cmpD_cc_regCF(rFlagsRegUCF cr, regD src1, regD src2) %{
13944 match(Set cr (CmpD src1 src2));
13945
13946 ins_cost(100);
13947 format %{ "ucomisd $src1, $src2 test" %}
13948 ins_encode %{
13949 __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
13950 %}
13951 ins_pipe(pipe_slow);
13952 %}
13953
13954 instruct cmpD_cc_regCFE(rFlagsRegUCFE cr, regD src1, regD src2) %{
13955 match(Set cr (CmpD src1 src2));
13956
13957 ins_cost(100);
13958 format %{ "evucomxsd $src1, $src2 test" %}
13959 ins_encode %{
13960 __ evucomxsd($src1$$XMMRegister, $src2$$XMMRegister);
13961 %}
13962 ins_pipe(pipe_slow);
13963 %}
13964
13965 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
13966 match(Set cr (CmpD src1 (LoadD src2)));
13967
13968 ins_cost(100);
13969 format %{ "ucomisd $src1, $src2" %}
13970 ins_encode %{
13971 __ ucomisd($src1$$XMMRegister, $src2$$Address);
13972 %}
13973 ins_pipe(pipe_slow);
13974 %}
13975
13976 instruct cmpD_cc_memCFE(rFlagsRegUCFE cr, regD src1, memory src2) %{
13977 match(Set cr (CmpD src1 (LoadD src2)));
13978
13979 ins_cost(100);
13980 format %{ "evucomxsd $src1, $src2" %}
13981 ins_encode %{
13982 __ evucomxsd($src1$$XMMRegister, $src2$$Address);
13983 %}
13984 ins_pipe(pipe_slow);
13985 %}
13986
13987 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
13988 match(Set cr (CmpD src con));
13989 ins_cost(100);
13990 format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
13991 ins_encode %{
13992 __ ucomisd($src$$XMMRegister, $constantaddress($con));
13993 %}
13994 ins_pipe(pipe_slow);
13995 %}
13996
13997 instruct cmpD_cc_immCFE(rFlagsRegUCFE cr, regD src, immD con) %{
13998 match(Set cr (CmpD src con));
13999
14000 ins_cost(100);
14001 format %{ "evucomxsd $src, [$constantaddress]\t# load from constant table: double=$con" %}
14002 ins_encode %{
14003 __ evucomxsd($src$$XMMRegister, $constantaddress($con));
14004 %}
14005 ins_pipe(pipe_slow);
14006 %}
14007
14008 // Compare into -1,0,1
14009 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
14010 %{
14011 match(Set dst (CmpF3 src1 src2));
14012 effect(KILL cr);
14013
14014 ins_cost(275);
14015 format %{ "ucomiss $src1, $src2\n\t"
14016 "movl $dst, #-1\n\t"
14017 "jp,s done\n\t"
14018 "jb,s done\n\t"
14019 "setne $dst\n\t"
14020 "movzbl $dst, $dst\n"
14021 "done:" %}
14022 ins_encode %{
14023 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
14024 emit_cmpfp3(masm, $dst$$Register);
14025 %}
14026 ins_pipe(pipe_slow);
14027 %}
14028
14029 // Compare into -1,0,1
14030 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
14031 %{
14032 match(Set dst (CmpF3 src1 (LoadF src2)));
14033 effect(KILL cr);
14034
14035 ins_cost(275);
14036 format %{ "ucomiss $src1, $src2\n\t"
14037 "movl $dst, #-1\n\t"
14038 "jp,s done\n\t"
14039 "jb,s done\n\t"
14040 "setne $dst\n\t"
14041 "movzbl $dst, $dst\n"
14042 "done:" %}
14043 ins_encode %{
14044 __ ucomiss($src1$$XMMRegister, $src2$$Address);
14045 emit_cmpfp3(masm, $dst$$Register);
14046 %}
14047 ins_pipe(pipe_slow);
14048 %}
14049
14050 // Compare into -1,0,1
14051 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
14052 match(Set dst (CmpF3 src con));
14053 effect(KILL cr);
14054
14055 ins_cost(275);
14056 format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
14057 "movl $dst, #-1\n\t"
14058 "jp,s done\n\t"
14059 "jb,s done\n\t"
14060 "setne $dst\n\t"
14061 "movzbl $dst, $dst\n"
14062 "done:" %}
14063 ins_encode %{
14064 __ ucomiss($src$$XMMRegister, $constantaddress($con));
14065 emit_cmpfp3(masm, $dst$$Register);
14066 %}
14067 ins_pipe(pipe_slow);
14068 %}
14069
14070 // Compare into -1,0,1
14071 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
14072 %{
14073 match(Set dst (CmpD3 src1 src2));
14074 effect(KILL cr);
14075
14076 ins_cost(275);
14077 format %{ "ucomisd $src1, $src2\n\t"
14078 "movl $dst, #-1\n\t"
14079 "jp,s done\n\t"
14080 "jb,s done\n\t"
14081 "setne $dst\n\t"
14082 "movzbl $dst, $dst\n"
14083 "done:" %}
14084 ins_encode %{
14085 __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
14086 emit_cmpfp3(masm, $dst$$Register);
14087 %}
14088 ins_pipe(pipe_slow);
14089 %}
14090
14091 // Compare into -1,0,1
14092 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
14093 %{
14094 match(Set dst (CmpD3 src1 (LoadD src2)));
14095 effect(KILL cr);
14096
14097 ins_cost(275);
14098 format %{ "ucomisd $src1, $src2\n\t"
14099 "movl $dst, #-1\n\t"
14100 "jp,s done\n\t"
14101 "jb,s done\n\t"
14102 "setne $dst\n\t"
14103 "movzbl $dst, $dst\n"
14104 "done:" %}
14105 ins_encode %{
14106 __ ucomisd($src1$$XMMRegister, $src2$$Address);
14107 emit_cmpfp3(masm, $dst$$Register);
14108 %}
14109 ins_pipe(pipe_slow);
14110 %}
14111
14112 // Compare into -1,0,1
14113 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
14114 match(Set dst (CmpD3 src con));
14115 effect(KILL cr);
14116
14117 ins_cost(275);
14118 format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
14119 "movl $dst, #-1\n\t"
14120 "jp,s done\n\t"
14121 "jb,s done\n\t"
14122 "setne $dst\n\t"
14123 "movzbl $dst, $dst\n"
14124 "done:" %}
14125 ins_encode %{
14126 __ ucomisd($src$$XMMRegister, $constantaddress($con));
14127 emit_cmpfp3(masm, $dst$$Register);
14128 %}
14129 ins_pipe(pipe_slow);
14130 %}
14131
14132 //----------Arithmetic Conversion Instructions---------------------------------
14133
14134 instruct convF2D_reg_reg(regD dst, regF src)
14135 %{
14136 match(Set dst (ConvF2D src));
14137
14138 format %{ "cvtss2sd $dst, $src" %}
14139 ins_encode %{
14140 __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
14141 %}
14142 ins_pipe(pipe_slow); // XXX
14143 %}
14144
14145 instruct convF2D_reg_mem(regD dst, memory src)
14146 %{
14147 predicate(UseAVX == 0);
14148 match(Set dst (ConvF2D (LoadF src)));
14149
14150 format %{ "cvtss2sd $dst, $src" %}
14151 ins_encode %{
14152 __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
14153 %}
14154 ins_pipe(pipe_slow); // XXX
14155 %}
14156
14157 instruct convD2F_reg_reg(regF dst, regD src)
14158 %{
14159 match(Set dst (ConvD2F src));
14160
14161 format %{ "cvtsd2ss $dst, $src" %}
14162 ins_encode %{
14163 __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
14164 %}
14165 ins_pipe(pipe_slow); // XXX
14166 %}
14167
14168 instruct convD2F_reg_mem(regF dst, memory src)
14169 %{
14170 predicate(UseAVX == 0);
14171 match(Set dst (ConvD2F (LoadD src)));
14172
14173 format %{ "cvtsd2ss $dst, $src" %}
14174 ins_encode %{
14175 __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
14176 %}
14177 ins_pipe(pipe_slow); // XXX
14178 %}
14179
14180 // XXX do mem variants
14181 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
14182 %{
14183 predicate(!VM_Version::supports_avx10_2());
14184 match(Set dst (ConvF2I src));
14185 effect(KILL cr);
14186 format %{ "convert_f2i $dst, $src" %}
14187 ins_encode %{
14188 __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
14189 %}
14190 ins_pipe(pipe_slow);
14191 %}
14192
14193 instruct convF2I_reg_reg_avx10_2(rRegI dst, regF src)
14194 %{
14195 predicate(VM_Version::supports_avx10_2());
14196 match(Set dst (ConvF2I src));
14197 format %{ "evcvttss2sisl $dst, $src" %}
14198 ins_encode %{
14199 __ evcvttss2sisl($dst$$Register, $src$$XMMRegister);
14200 %}
14201 ins_pipe(pipe_slow);
14202 %}
14203
14204 instruct convF2I_reg_mem_avx10_2(rRegI dst, memory src)
14205 %{
14206 predicate(VM_Version::supports_avx10_2());
14207 match(Set dst (ConvF2I (LoadF src)));
14208 format %{ "evcvttss2sisl $dst, $src" %}
14209 ins_encode %{
14210 __ evcvttss2sisl($dst$$Register, $src$$Address);
14211 %}
14212 ins_pipe(pipe_slow);
14213 %}
14214
14215 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
14216 %{
14217 predicate(!VM_Version::supports_avx10_2());
14218 match(Set dst (ConvF2L src));
14219 effect(KILL cr);
14220 format %{ "convert_f2l $dst, $src"%}
14221 ins_encode %{
14222 __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
14223 %}
14224 ins_pipe(pipe_slow);
14225 %}
14226
14227 instruct convF2L_reg_reg_avx10_2(rRegL dst, regF src)
14228 %{
14229 predicate(VM_Version::supports_avx10_2());
14230 match(Set dst (ConvF2L src));
14231 format %{ "evcvttss2sisq $dst, $src" %}
14232 ins_encode %{
14233 __ evcvttss2sisq($dst$$Register, $src$$XMMRegister);
14234 %}
14235 ins_pipe(pipe_slow);
14236 %}
14237
14238 instruct convF2L_reg_mem_avx10_2(rRegL dst, memory src)
14239 %{
14240 predicate(VM_Version::supports_avx10_2());
14241 match(Set dst (ConvF2L (LoadF src)));
14242 format %{ "evcvttss2sisq $dst, $src" %}
14243 ins_encode %{
14244 __ evcvttss2sisq($dst$$Register, $src$$Address);
14245 %}
14246 ins_pipe(pipe_slow);
14247 %}
14248
14249 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
14250 %{
14251 predicate(!VM_Version::supports_avx10_2());
14252 match(Set dst (ConvD2I src));
14253 effect(KILL cr);
14254 format %{ "convert_d2i $dst, $src"%}
14255 ins_encode %{
14256 __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
14257 %}
14258 ins_pipe(pipe_slow);
14259 %}
14260
14261 instruct convD2I_reg_reg_avx10_2(rRegI dst, regD src)
14262 %{
14263 predicate(VM_Version::supports_avx10_2());
14264 match(Set dst (ConvD2I src));
14265 format %{ "evcvttsd2sisl $dst, $src" %}
14266 ins_encode %{
14267 __ evcvttsd2sisl($dst$$Register, $src$$XMMRegister);
14268 %}
14269 ins_pipe(pipe_slow);
14270 %}
14271
14272 instruct convD2I_reg_mem_avx10_2(rRegI dst, memory src)
14273 %{
14274 predicate(VM_Version::supports_avx10_2());
14275 match(Set dst (ConvD2I (LoadD src)));
14276 format %{ "evcvttsd2sisl $dst, $src" %}
14277 ins_encode %{
14278 __ evcvttsd2sisl($dst$$Register, $src$$Address);
14279 %}
14280 ins_pipe(pipe_slow);
14281 %}
14282
14283 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
14284 %{
14285 predicate(!VM_Version::supports_avx10_2());
14286 match(Set dst (ConvD2L src));
14287 effect(KILL cr);
14288 format %{ "convert_d2l $dst, $src"%}
14289 ins_encode %{
14290 __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
14291 %}
14292 ins_pipe(pipe_slow);
14293 %}
14294
14295 instruct convD2L_reg_reg_avx10_2(rRegL dst, regD src)
14296 %{
14297 predicate(VM_Version::supports_avx10_2());
14298 match(Set dst (ConvD2L src));
14299 format %{ "evcvttsd2sisq $dst, $src" %}
14300 ins_encode %{
14301 __ evcvttsd2sisq($dst$$Register, $src$$XMMRegister);
14302 %}
14303 ins_pipe(pipe_slow);
14304 %}
14305
14306 instruct convD2L_reg_mem_avx10_2(rRegL dst, memory src)
14307 %{
14308 predicate(VM_Version::supports_avx10_2());
14309 match(Set dst (ConvD2L (LoadD src)));
14310 format %{ "evcvttsd2sisq $dst, $src" %}
14311 ins_encode %{
14312 __ evcvttsd2sisq($dst$$Register, $src$$Address);
14313 %}
14314 ins_pipe(pipe_slow);
14315 %}
14316
14317 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
14318 %{
14319 match(Set dst (RoundD src));
14320 effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
14321 format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
14322 ins_encode %{
14323 __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
14324 %}
14325 ins_pipe(pipe_slow);
14326 %}
14327
14328 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
14329 %{
14330 match(Set dst (RoundF src));
14331 effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
14332 format %{ "round_float $dst,$src" %}
14333 ins_encode %{
14334 __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
14335 %}
14336 ins_pipe(pipe_slow);
14337 %}
14338
14339 instruct convI2F_reg_reg(vlRegF dst, rRegI src)
14340 %{
14341 predicate(!UseXmmI2F);
14342 match(Set dst (ConvI2F src));
14343
14344 format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
14345 ins_encode %{
14346 if (UseAVX > 0) {
14347 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14348 }
14349 __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
14350 %}
14351 ins_pipe(pipe_slow); // XXX
14352 %}
14353
14354 instruct convI2F_reg_mem(regF dst, memory src)
14355 %{
14356 predicate(UseAVX == 0);
14357 match(Set dst (ConvI2F (LoadI src)));
14358
14359 format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
14360 ins_encode %{
14361 __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
14362 %}
14363 ins_pipe(pipe_slow); // XXX
14364 %}
14365
14366 instruct convI2D_reg_reg(vlRegD dst, rRegI src)
14367 %{
14368 predicate(!UseXmmI2D);
14369 match(Set dst (ConvI2D src));
14370
14371 format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
14372 ins_encode %{
14373 if (UseAVX > 0) {
14374 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14375 }
14376 __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
14377 %}
14378 ins_pipe(pipe_slow); // XXX
14379 %}
14380
14381 instruct convI2D_reg_mem(regD dst, memory src)
14382 %{
14383 predicate(UseAVX == 0);
14384 match(Set dst (ConvI2D (LoadI src)));
14385
14386 format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
14387 ins_encode %{
14388 __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
14389 %}
14390 ins_pipe(pipe_slow); // XXX
14391 %}
14392
14393 instruct convXI2F_reg(regF dst, rRegI src)
14394 %{
14395 predicate(UseXmmI2F);
14396 match(Set dst (ConvI2F src));
14397
14398 format %{ "movdl $dst, $src\n\t"
14399 "cvtdq2psl $dst, $dst\t# i2f" %}
14400 ins_encode %{
14401 __ movdl($dst$$XMMRegister, $src$$Register);
14402 __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
14403 %}
14404 ins_pipe(pipe_slow); // XXX
14405 %}
14406
14407 instruct convXI2D_reg(regD dst, rRegI src)
14408 %{
14409 predicate(UseXmmI2D);
14410 match(Set dst (ConvI2D src));
14411
14412 format %{ "movdl $dst, $src\n\t"
14413 "cvtdq2pdl $dst, $dst\t# i2d" %}
14414 ins_encode %{
14415 __ movdl($dst$$XMMRegister, $src$$Register);
14416 __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
14417 %}
14418 ins_pipe(pipe_slow); // XXX
14419 %}
14420
14421 instruct convL2F_reg_reg(vlRegF dst, rRegL src)
14422 %{
14423 match(Set dst (ConvL2F src));
14424
14425 format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
14426 ins_encode %{
14427 if (UseAVX > 0) {
14428 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14429 }
14430 __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
14431 %}
14432 ins_pipe(pipe_slow); // XXX
14433 %}
14434
14435 instruct convL2F_reg_mem(regF dst, memory src)
14436 %{
14437 predicate(UseAVX == 0);
14438 match(Set dst (ConvL2F (LoadL src)));
14439
14440 format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
14441 ins_encode %{
14442 __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
14443 %}
14444 ins_pipe(pipe_slow); // XXX
14445 %}
14446
14447 instruct convL2D_reg_reg(vlRegD dst, rRegL src)
14448 %{
14449 match(Set dst (ConvL2D src));
14450
14451 format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
14452 ins_encode %{
14453 if (UseAVX > 0) {
14454 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
14455 }
14456 __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
14457 %}
14458 ins_pipe(pipe_slow); // XXX
14459 %}
14460
14461 instruct convL2D_reg_mem(regD dst, memory src)
14462 %{
14463 predicate(UseAVX == 0);
14464 match(Set dst (ConvL2D (LoadL src)));
14465
14466 format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
14467 ins_encode %{
14468 __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
14469 %}
14470 ins_pipe(pipe_slow); // XXX
14471 %}
14472
14473 instruct convI2L_reg_reg(rRegL dst, rRegI src)
14474 %{
14475 match(Set dst (ConvI2L src));
14476
14477 ins_cost(125);
14478 format %{ "movslq $dst, $src\t# i2l" %}
14479 ins_encode %{
14480 __ movslq($dst$$Register, $src$$Register);
14481 %}
14482 ins_pipe(ialu_reg_reg);
14483 %}
14484
14485 // Zero-extend convert int to long
14486 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
14487 %{
14488 match(Set dst (AndL (ConvI2L src) mask));
14489
14490 format %{ "movl $dst, $src\t# i2l zero-extend\n\t" %}
14491 ins_encode %{
14492 if ($dst$$reg != $src$$reg) {
14493 __ movl($dst$$Register, $src$$Register);
14494 }
14495 %}
14496 ins_pipe(ialu_reg_reg);
14497 %}
14498
14499 // Zero-extend convert int to long
14500 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
14501 %{
14502 match(Set dst (AndL (ConvI2L (LoadI src)) mask));
14503
14504 format %{ "movl $dst, $src\t# i2l zero-extend\n\t" %}
14505 ins_encode %{
14506 __ movl($dst$$Register, $src$$Address);
14507 %}
14508 ins_pipe(ialu_reg_mem);
14509 %}
14510
14511 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
14512 %{
14513 match(Set dst (AndL src mask));
14514
14515 format %{ "movl $dst, $src\t# zero-extend long" %}
14516 ins_encode %{
14517 __ movl($dst$$Register, $src$$Register);
14518 %}
14519 ins_pipe(ialu_reg_reg);
14520 %}
14521
14522 instruct convL2I_reg_reg(rRegI dst, rRegL src)
14523 %{
14524 match(Set dst (ConvL2I src));
14525
14526 format %{ "movl $dst, $src\t# l2i" %}
14527 ins_encode %{
14528 __ movl($dst$$Register, $src$$Register);
14529 %}
14530 ins_pipe(ialu_reg_reg);
14531 %}
14532
14533
14534 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
14535 match(Set dst (MoveF2I src));
14536 effect(DEF dst, USE src);
14537
14538 ins_cost(125);
14539 format %{ "movl $dst, $src\t# MoveF2I_stack_reg" %}
14540 ins_encode %{
14541 __ movl($dst$$Register, Address(rsp, $src$$disp));
14542 %}
14543 ins_pipe(ialu_reg_mem);
14544 %}
14545
14546 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
14547 match(Set dst (MoveI2F src));
14548 effect(DEF dst, USE src);
14549
14550 ins_cost(125);
14551 format %{ "movss $dst, $src\t# MoveI2F_stack_reg" %}
14552 ins_encode %{
14553 __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
14554 %}
14555 ins_pipe(pipe_slow);
14556 %}
14557
14558 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
14559 match(Set dst (MoveD2L src));
14560 effect(DEF dst, USE src);
14561
14562 ins_cost(125);
14563 format %{ "movq $dst, $src\t# MoveD2L_stack_reg" %}
14564 ins_encode %{
14565 __ movq($dst$$Register, Address(rsp, $src$$disp));
14566 %}
14567 ins_pipe(ialu_reg_mem);
14568 %}
14569
14570 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
14571 predicate(!UseXmmLoadAndClearUpper);
14572 match(Set dst (MoveL2D src));
14573 effect(DEF dst, USE src);
14574
14575 ins_cost(125);
14576 format %{ "movlpd $dst, $src\t# MoveL2D_stack_reg" %}
14577 ins_encode %{
14578 __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
14579 %}
14580 ins_pipe(pipe_slow);
14581 %}
14582
14583 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
14584 predicate(UseXmmLoadAndClearUpper);
14585 match(Set dst (MoveL2D src));
14586 effect(DEF dst, USE src);
14587
14588 ins_cost(125);
14589 format %{ "movsd $dst, $src\t# MoveL2D_stack_reg" %}
14590 ins_encode %{
14591 __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
14592 %}
14593 ins_pipe(pipe_slow);
14594 %}
14595
14596
14597 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
14598 match(Set dst (MoveF2I src));
14599 effect(DEF dst, USE src);
14600
14601 ins_cost(95); // XXX
14602 format %{ "movss $dst, $src\t# MoveF2I_reg_stack" %}
14603 ins_encode %{
14604 __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
14605 %}
14606 ins_pipe(pipe_slow);
14607 %}
14608
14609 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
14610 match(Set dst (MoveI2F src));
14611 effect(DEF dst, USE src);
14612
14613 ins_cost(100);
14614 format %{ "movl $dst, $src\t# MoveI2F_reg_stack" %}
14615 ins_encode %{
14616 __ movl(Address(rsp, $dst$$disp), $src$$Register);
14617 %}
14618 ins_pipe( ialu_mem_reg );
14619 %}
14620
14621 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
14622 match(Set dst (MoveD2L src));
14623 effect(DEF dst, USE src);
14624
14625 ins_cost(95); // XXX
14626 format %{ "movsd $dst, $src\t# MoveL2D_reg_stack" %}
14627 ins_encode %{
14628 __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
14629 %}
14630 ins_pipe(pipe_slow);
14631 %}
14632
14633 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
14634 match(Set dst (MoveL2D src));
14635 effect(DEF dst, USE src);
14636
14637 ins_cost(100);
14638 format %{ "movq $dst, $src\t# MoveL2D_reg_stack" %}
14639 ins_encode %{
14640 __ movq(Address(rsp, $dst$$disp), $src$$Register);
14641 %}
14642 ins_pipe(ialu_mem_reg);
14643 %}
14644
14645 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
14646 match(Set dst (MoveF2I src));
14647 effect(DEF dst, USE src);
14648 ins_cost(85);
14649 format %{ "movd $dst,$src\t# MoveF2I" %}
14650 ins_encode %{
14651 __ movdl($dst$$Register, $src$$XMMRegister);
14652 %}
14653 ins_pipe( pipe_slow );
14654 %}
14655
14656 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
14657 match(Set dst (MoveD2L src));
14658 effect(DEF dst, USE src);
14659 ins_cost(85);
14660 format %{ "movd $dst,$src\t# MoveD2L" %}
14661 ins_encode %{
14662 __ movdq($dst$$Register, $src$$XMMRegister);
14663 %}
14664 ins_pipe( pipe_slow );
14665 %}
14666
14667 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
14668 match(Set dst (MoveI2F src));
14669 effect(DEF dst, USE src);
14670 ins_cost(100);
14671 format %{ "movd $dst,$src\t# MoveI2F" %}
14672 ins_encode %{
14673 __ movdl($dst$$XMMRegister, $src$$Register);
14674 %}
14675 ins_pipe( pipe_slow );
14676 %}
14677
14678 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
14679 match(Set dst (MoveL2D src));
14680 effect(DEF dst, USE src);
14681 ins_cost(100);
14682 format %{ "movd $dst,$src\t# MoveL2D" %}
14683 ins_encode %{
14684 __ movdq($dst$$XMMRegister, $src$$Register);
14685 %}
14686 ins_pipe( pipe_slow );
14687 %}
14688
14689 // Fast clearing of an array
14690 // Small non-constant lenght ClearArray for non-AVX512 targets.
14691 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
14692 Universe dummy, rFlagsReg cr)
14693 %{
14694 predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
14695 match(Set dummy (ClearArray cnt base));
14696 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
14697
14698 format %{ $$template
14699 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14700 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14701 $$emit$$"jg LARGE\n\t"
14702 $$emit$$"dec rcx\n\t"
14703 $$emit$$"js DONE\t# Zero length\n\t"
14704 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14705 $$emit$$"dec rcx\n\t"
14706 $$emit$$"jge LOOP\n\t"
14707 $$emit$$"jmp DONE\n\t"
14708 $$emit$$"# LARGE:\n\t"
14709 if (UseFastStosb) {
14710 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14711 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14712 } else if (UseXMMForObjInit) {
14713 $$emit$$"mov rdi,rax\n\t"
14714 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14715 $$emit$$"jmpq L_zero_64_bytes\n\t"
14716 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14717 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14718 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14719 $$emit$$"add 0x40,rax\n\t"
14720 $$emit$$"# L_zero_64_bytes:\n\t"
14721 $$emit$$"sub 0x8,rcx\n\t"
14722 $$emit$$"jge L_loop\n\t"
14723 $$emit$$"add 0x4,rcx\n\t"
14724 $$emit$$"jl L_tail\n\t"
14725 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14726 $$emit$$"add 0x20,rax\n\t"
14727 $$emit$$"sub 0x4,rcx\n\t"
14728 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14729 $$emit$$"add 0x4,rcx\n\t"
14730 $$emit$$"jle L_end\n\t"
14731 $$emit$$"dec rcx\n\t"
14732 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14733 $$emit$$"vmovq xmm0,(rax)\n\t"
14734 $$emit$$"add 0x8,rax\n\t"
14735 $$emit$$"dec rcx\n\t"
14736 $$emit$$"jge L_sloop\n\t"
14737 $$emit$$"# L_end:\n\t"
14738 } else {
14739 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14740 }
14741 $$emit$$"# DONE"
14742 %}
14743 ins_encode %{
14744 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14745 $tmp$$XMMRegister, false, knoreg);
14746 %}
14747 ins_pipe(pipe_slow);
14748 %}
14749
14750 // Small non-constant length ClearArray for AVX512 targets.
14751 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
14752 Universe dummy, rFlagsReg cr)
14753 %{
14754 predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
14755 match(Set dummy (ClearArray cnt base));
14756 ins_cost(125);
14757 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
14758
14759 format %{ $$template
14760 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14761 $$emit$$"cmp InitArrayShortSize,rcx\n\t"
14762 $$emit$$"jg LARGE\n\t"
14763 $$emit$$"dec rcx\n\t"
14764 $$emit$$"js DONE\t# Zero length\n\t"
14765 $$emit$$"mov rax,(rdi,rcx,8)\t# LOOP\n\t"
14766 $$emit$$"dec rcx\n\t"
14767 $$emit$$"jge LOOP\n\t"
14768 $$emit$$"jmp DONE\n\t"
14769 $$emit$$"# LARGE:\n\t"
14770 if (UseFastStosb) {
14771 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14772 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--\n\t"
14773 } else if (UseXMMForObjInit) {
14774 $$emit$$"mov rdi,rax\n\t"
14775 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14776 $$emit$$"jmpq L_zero_64_bytes\n\t"
14777 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14778 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14779 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14780 $$emit$$"add 0x40,rax\n\t"
14781 $$emit$$"# L_zero_64_bytes:\n\t"
14782 $$emit$$"sub 0x8,rcx\n\t"
14783 $$emit$$"jge L_loop\n\t"
14784 $$emit$$"add 0x4,rcx\n\t"
14785 $$emit$$"jl L_tail\n\t"
14786 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14787 $$emit$$"add 0x20,rax\n\t"
14788 $$emit$$"sub 0x4,rcx\n\t"
14789 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14790 $$emit$$"add 0x4,rcx\n\t"
14791 $$emit$$"jle L_end\n\t"
14792 $$emit$$"dec rcx\n\t"
14793 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14794 $$emit$$"vmovq xmm0,(rax)\n\t"
14795 $$emit$$"add 0x8,rax\n\t"
14796 $$emit$$"dec rcx\n\t"
14797 $$emit$$"jge L_sloop\n\t"
14798 $$emit$$"# L_end:\n\t"
14799 } else {
14800 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--\n\t"
14801 }
14802 $$emit$$"# DONE"
14803 %}
14804 ins_encode %{
14805 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14806 $tmp$$XMMRegister, false, $ktmp$$KRegister);
14807 %}
14808 ins_pipe(pipe_slow);
14809 %}
14810
14811 // Large non-constant length ClearArray for non-AVX512 targets.
14812 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
14813 Universe dummy, rFlagsReg cr)
14814 %{
14815 predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
14816 match(Set dummy (ClearArray cnt base));
14817 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
14818
14819 format %{ $$template
14820 if (UseFastStosb) {
14821 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14822 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14823 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
14824 } else if (UseXMMForObjInit) {
14825 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
14826 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14827 $$emit$$"jmpq L_zero_64_bytes\n\t"
14828 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14829 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14830 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14831 $$emit$$"add 0x40,rax\n\t"
14832 $$emit$$"# L_zero_64_bytes:\n\t"
14833 $$emit$$"sub 0x8,rcx\n\t"
14834 $$emit$$"jge L_loop\n\t"
14835 $$emit$$"add 0x4,rcx\n\t"
14836 $$emit$$"jl L_tail\n\t"
14837 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14838 $$emit$$"add 0x20,rax\n\t"
14839 $$emit$$"sub 0x4,rcx\n\t"
14840 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14841 $$emit$$"add 0x4,rcx\n\t"
14842 $$emit$$"jle L_end\n\t"
14843 $$emit$$"dec rcx\n\t"
14844 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14845 $$emit$$"vmovq xmm0,(rax)\n\t"
14846 $$emit$$"add 0x8,rax\n\t"
14847 $$emit$$"dec rcx\n\t"
14848 $$emit$$"jge L_sloop\n\t"
14849 $$emit$$"# L_end:\n\t"
14850 } else {
14851 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14852 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
14853 }
14854 %}
14855 ins_encode %{
14856 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14857 $tmp$$XMMRegister, true, knoreg);
14858 %}
14859 ins_pipe(pipe_slow);
14860 %}
14861
14862 // Large non-constant length ClearArray for AVX512 targets.
14863 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
14864 Universe dummy, rFlagsReg cr)
14865 %{
14866 predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
14867 match(Set dummy (ClearArray cnt base));
14868 effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
14869
14870 format %{ $$template
14871 if (UseFastStosb) {
14872 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14873 $$emit$$"shlq rcx,3\t# Convert doublewords to bytes\n\t"
14874 $$emit$$"rep stosb\t# Store rax to *rdi++ while rcx--"
14875 } else if (UseXMMForObjInit) {
14876 $$emit$$"mov rdi,rax\t# ClearArray:\n\t"
14877 $$emit$$"vpxor ymm0,ymm0,ymm0\n\t"
14878 $$emit$$"jmpq L_zero_64_bytes\n\t"
14879 $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
14880 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14881 $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
14882 $$emit$$"add 0x40,rax\n\t"
14883 $$emit$$"# L_zero_64_bytes:\n\t"
14884 $$emit$$"sub 0x8,rcx\n\t"
14885 $$emit$$"jge L_loop\n\t"
14886 $$emit$$"add 0x4,rcx\n\t"
14887 $$emit$$"jl L_tail\n\t"
14888 $$emit$$"vmovdqu ymm0,(rax)\n\t"
14889 $$emit$$"add 0x20,rax\n\t"
14890 $$emit$$"sub 0x4,rcx\n\t"
14891 $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
14892 $$emit$$"add 0x4,rcx\n\t"
14893 $$emit$$"jle L_end\n\t"
14894 $$emit$$"dec rcx\n\t"
14895 $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
14896 $$emit$$"vmovq xmm0,(rax)\n\t"
14897 $$emit$$"add 0x8,rax\n\t"
14898 $$emit$$"dec rcx\n\t"
14899 $$emit$$"jge L_sloop\n\t"
14900 $$emit$$"# L_end:\n\t"
14901 } else {
14902 $$emit$$"xorq rax, rax\t# ClearArray:\n\t"
14903 $$emit$$"rep stosq\t# Store rax to *rdi++ while rcx--"
14904 }
14905 %}
14906 ins_encode %{
14907 __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
14908 $tmp$$XMMRegister, true, $ktmp$$KRegister);
14909 %}
14910 ins_pipe(pipe_slow);
14911 %}
14912
14913 // Small constant length ClearArray for AVX512 targets.
14914 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
14915 %{
14916 predicate(!((ClearArrayNode*)n)->is_large() && (MaxVectorSize >= 32) && VM_Version::supports_avx512vl());
14917 match(Set dummy (ClearArray cnt base));
14918 ins_cost(100);
14919 effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
14920 format %{ "clear_mem_imm $base , $cnt \n\t" %}
14921 ins_encode %{
14922 __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
14923 %}
14924 ins_pipe(pipe_slow);
14925 %}
14926
14927 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
14928 rax_RegI result, legRegD tmp1, rFlagsReg cr)
14929 %{
14930 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
14931 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14932 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14933
14934 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
14935 ins_encode %{
14936 __ string_compare($str1$$Register, $str2$$Register,
14937 $cnt1$$Register, $cnt2$$Register, $result$$Register,
14938 $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
14939 %}
14940 ins_pipe( pipe_slow );
14941 %}
14942
14943 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
14944 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
14945 %{
14946 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
14947 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14948 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14949
14950 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
14951 ins_encode %{
14952 __ string_compare($str1$$Register, $str2$$Register,
14953 $cnt1$$Register, $cnt2$$Register, $result$$Register,
14954 $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
14955 %}
14956 ins_pipe( pipe_slow );
14957 %}
14958
14959 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
14960 rax_RegI result, legRegD tmp1, rFlagsReg cr)
14961 %{
14962 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
14963 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14964 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14965
14966 format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
14967 ins_encode %{
14968 __ string_compare($str1$$Register, $str2$$Register,
14969 $cnt1$$Register, $cnt2$$Register, $result$$Register,
14970 $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
14971 %}
14972 ins_pipe( pipe_slow );
14973 %}
14974
14975 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
14976 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
14977 %{
14978 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
14979 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14980 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14981
14982 format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
14983 ins_encode %{
14984 __ string_compare($str1$$Register, $str2$$Register,
14985 $cnt1$$Register, $cnt2$$Register, $result$$Register,
14986 $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
14987 %}
14988 ins_pipe( pipe_slow );
14989 %}
14990
14991 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
14992 rax_RegI result, legRegD tmp1, rFlagsReg cr)
14993 %{
14994 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
14995 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
14996 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
14997
14998 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
14999 ins_encode %{
15000 __ string_compare($str1$$Register, $str2$$Register,
15001 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15002 $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
15003 %}
15004 ins_pipe( pipe_slow );
15005 %}
15006
15007 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
15008 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
15009 %{
15010 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
15011 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15012 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15013
15014 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15015 ins_encode %{
15016 __ string_compare($str1$$Register, $str2$$Register,
15017 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15018 $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
15019 %}
15020 ins_pipe( pipe_slow );
15021 %}
15022
15023 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
15024 rax_RegI result, legRegD tmp1, rFlagsReg cr)
15025 %{
15026 predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
15027 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15028 effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15029
15030 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15031 ins_encode %{
15032 __ string_compare($str2$$Register, $str1$$Register,
15033 $cnt2$$Register, $cnt1$$Register, $result$$Register,
15034 $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
15035 %}
15036 ins_pipe( pipe_slow );
15037 %}
15038
15039 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
15040 rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
15041 %{
15042 predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
15043 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15044 effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15045
15046 format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
15047 ins_encode %{
15048 __ string_compare($str2$$Register, $str1$$Register,
15049 $cnt2$$Register, $cnt1$$Register, $result$$Register,
15050 $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
15051 %}
15052 ins_pipe( pipe_slow );
15053 %}
15054
15055 // fast search of substring with known size.
15056 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
15057 rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
15058 %{
15059 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
15060 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15061 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
15062
15063 format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
15064 ins_encode %{
15065 int icnt2 = (int)$int_cnt2$$constant;
15066 if (icnt2 >= 16) {
15067 // IndexOf for constant substrings with size >= 16 elements
15068 // which don't need to be loaded through stack.
15069 __ string_indexofC8($str1$$Register, $str2$$Register,
15070 $cnt1$$Register, $cnt2$$Register,
15071 icnt2, $result$$Register,
15072 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
15073 } else {
15074 // Small strings are loaded through stack if they cross page boundary.
15075 __ string_indexof($str1$$Register, $str2$$Register,
15076 $cnt1$$Register, $cnt2$$Register,
15077 icnt2, $result$$Register,
15078 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
15079 }
15080 %}
15081 ins_pipe( pipe_slow );
15082 %}
15083
15084 // fast search of substring with known size.
15085 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
15086 rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
15087 %{
15088 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
15089 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15090 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
15091
15092 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
15093 ins_encode %{
15094 int icnt2 = (int)$int_cnt2$$constant;
15095 if (icnt2 >= 8) {
15096 // IndexOf for constant substrings with size >= 8 elements
15097 // which don't need to be loaded through stack.
15098 __ string_indexofC8($str1$$Register, $str2$$Register,
15099 $cnt1$$Register, $cnt2$$Register,
15100 icnt2, $result$$Register,
15101 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
15102 } else {
15103 // Small strings are loaded through stack if they cross page boundary.
15104 __ string_indexof($str1$$Register, $str2$$Register,
15105 $cnt1$$Register, $cnt2$$Register,
15106 icnt2, $result$$Register,
15107 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
15108 }
15109 %}
15110 ins_pipe( pipe_slow );
15111 %}
15112
15113 // fast search of substring with known size.
15114 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
15115 rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
15116 %{
15117 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
15118 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15119 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
15120
15121 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
15122 ins_encode %{
15123 int icnt2 = (int)$int_cnt2$$constant;
15124 if (icnt2 >= 8) {
15125 // IndexOf for constant substrings with size >= 8 elements
15126 // which don't need to be loaded through stack.
15127 __ string_indexofC8($str1$$Register, $str2$$Register,
15128 $cnt1$$Register, $cnt2$$Register,
15129 icnt2, $result$$Register,
15130 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
15131 } else {
15132 // Small strings are loaded through stack if they cross page boundary.
15133 __ string_indexof($str1$$Register, $str2$$Register,
15134 $cnt1$$Register, $cnt2$$Register,
15135 icnt2, $result$$Register,
15136 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
15137 }
15138 %}
15139 ins_pipe( pipe_slow );
15140 %}
15141
15142 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
15143 rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
15144 %{
15145 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
15146 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15147 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
15148
15149 format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
15150 ins_encode %{
15151 __ string_indexof($str1$$Register, $str2$$Register,
15152 $cnt1$$Register, $cnt2$$Register,
15153 (-1), $result$$Register,
15154 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
15155 %}
15156 ins_pipe( pipe_slow );
15157 %}
15158
15159 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
15160 rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
15161 %{
15162 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
15163 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15164 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
15165
15166 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
15167 ins_encode %{
15168 __ string_indexof($str1$$Register, $str2$$Register,
15169 $cnt1$$Register, $cnt2$$Register,
15170 (-1), $result$$Register,
15171 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
15172 %}
15173 ins_pipe( pipe_slow );
15174 %}
15175
15176 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
15177 rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
15178 %{
15179 predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
15180 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15181 effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
15182
15183 format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
15184 ins_encode %{
15185 __ string_indexof($str1$$Register, $str2$$Register,
15186 $cnt1$$Register, $cnt2$$Register,
15187 (-1), $result$$Register,
15188 $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
15189 %}
15190 ins_pipe( pipe_slow );
15191 %}
15192
15193 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
15194 rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
15195 %{
15196 predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15197 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15198 effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
15199 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result // KILL all" %}
15200 ins_encode %{
15201 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
15202 $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
15203 %}
15204 ins_pipe( pipe_slow );
15205 %}
15206
15207 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
15208 rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
15209 %{
15210 predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15211 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15212 effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
15213 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result // KILL all" %}
15214 ins_encode %{
15215 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
15216 $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
15217 %}
15218 ins_pipe( pipe_slow );
15219 %}
15220
15221 // fast string equals
15222 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
15223 legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
15224 %{
15225 predicate(!VM_Version::supports_avx512vlbw());
15226 match(Set result (StrEquals (Binary str1 str2) cnt));
15227 effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
15228
15229 format %{ "String Equals $str1,$str2,$cnt -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15230 ins_encode %{
15231 __ arrays_equals(false, $str1$$Register, $str2$$Register,
15232 $cnt$$Register, $result$$Register, $tmp3$$Register,
15233 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
15234 %}
15235 ins_pipe( pipe_slow );
15236 %}
15237
15238 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
15239 legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
15240 %{
15241 predicate(VM_Version::supports_avx512vlbw());
15242 match(Set result (StrEquals (Binary str1 str2) cnt));
15243 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
15244
15245 format %{ "String Equals $str1,$str2,$cnt -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15246 ins_encode %{
15247 __ arrays_equals(false, $str1$$Register, $str2$$Register,
15248 $cnt$$Register, $result$$Register, $tmp3$$Register,
15249 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
15250 %}
15251 ins_pipe( pipe_slow );
15252 %}
15253
15254 // fast array equals
15255 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15256 legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15257 %{
15258 predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
15259 match(Set result (AryEq ary1 ary2));
15260 effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15261
15262 format %{ "Array Equals byte[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15263 ins_encode %{
15264 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15265 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15266 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
15267 %}
15268 ins_pipe( pipe_slow );
15269 %}
15270
15271 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15272 legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15273 %{
15274 predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
15275 match(Set result (AryEq ary1 ary2));
15276 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15277
15278 format %{ "Array Equals byte[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15279 ins_encode %{
15280 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15281 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15282 $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
15283 %}
15284 ins_pipe( pipe_slow );
15285 %}
15286
15287 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15288 legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15289 %{
15290 predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
15291 match(Set result (AryEq ary1 ary2));
15292 effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15293
15294 format %{ "Array Equals char[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15295 ins_encode %{
15296 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15297 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15298 $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
15299 %}
15300 ins_pipe( pipe_slow );
15301 %}
15302
15303 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
15304 legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
15305 %{
15306 predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
15307 match(Set result (AryEq ary1 ary2));
15308 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
15309
15310 format %{ "Array Equals char[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
15311 ins_encode %{
15312 __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
15313 $tmp3$$Register, $result$$Register, $tmp4$$Register,
15314 $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
15315 %}
15316 ins_pipe( pipe_slow );
15317 %}
15318
15319 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
15320 legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
15321 legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
15322 legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
15323 legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
15324 %{
15325 predicate(UseAVX >= 2);
15326 match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
15327 effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
15328 TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
15329 TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
15330 USE basic_type, KILL cr);
15331
15332 format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result // KILL all" %}
15333 ins_encode %{
15334 __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
15335 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
15336 $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
15337 $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
15338 $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
15339 $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
15340 $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
15341 %}
15342 ins_pipe( pipe_slow );
15343 %}
15344
15345 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
15346 legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
15347 %{
15348 predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
15349 match(Set result (CountPositives ary1 len));
15350 effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
15351
15352 format %{ "countPositives byte[] $ary1,$len -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15353 ins_encode %{
15354 __ count_positives($ary1$$Register, $len$$Register,
15355 $result$$Register, $tmp3$$Register,
15356 $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
15357 %}
15358 ins_pipe( pipe_slow );
15359 %}
15360
15361 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
15362 legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
15363 %{
15364 predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
15365 match(Set result (CountPositives ary1 len));
15366 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
15367
15368 format %{ "countPositives byte[] $ary1,$len -> $result // KILL $tmp1, $tmp2, $tmp3" %}
15369 ins_encode %{
15370 __ count_positives($ary1$$Register, $len$$Register,
15371 $result$$Register, $tmp3$$Register,
15372 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
15373 %}
15374 ins_pipe( pipe_slow );
15375 %}
15376
15377 // fast char[] to byte[] compression
15378 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
15379 legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15380 predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
15381 match(Set result (StrCompressedCopy src (Binary dst len)));
15382 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
15383 USE_KILL len, KILL tmp5, KILL cr);
15384
15385 format %{ "String Compress $src,$dst -> $result // KILL RAX, RCX, RDX" %}
15386 ins_encode %{
15387 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
15388 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15389 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
15390 knoreg, knoreg);
15391 %}
15392 ins_pipe( pipe_slow );
15393 %}
15394
15395 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
15396 legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15397 predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
15398 match(Set result (StrCompressedCopy src (Binary dst len)));
15399 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
15400 USE_KILL len, KILL tmp5, KILL cr);
15401
15402 format %{ "String Compress $src,$dst -> $result // KILL RAX, RCX, RDX" %}
15403 ins_encode %{
15404 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
15405 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15406 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
15407 $ktmp1$$KRegister, $ktmp2$$KRegister);
15408 %}
15409 ins_pipe( pipe_slow );
15410 %}
15411 // fast byte[] to char[] inflation
15412 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15413 legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
15414 predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
15415 match(Set dummy (StrInflatedCopy src (Binary dst len)));
15416 effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
15417
15418 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
15419 ins_encode %{
15420 __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
15421 $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
15422 %}
15423 ins_pipe( pipe_slow );
15424 %}
15425
15426 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15427 legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
15428 predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
15429 match(Set dummy (StrInflatedCopy src (Binary dst len)));
15430 effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
15431
15432 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
15433 ins_encode %{
15434 __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
15435 $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
15436 %}
15437 ins_pipe( pipe_slow );
15438 %}
15439
15440 // encode char[] to byte[] in ISO_8859_1
15441 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15442 legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
15443 rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15444 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
15445 match(Set result (EncodeISOArray src (Binary dst len)));
15446 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
15447
15448 format %{ "Encode iso array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
15449 ins_encode %{
15450 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
15451 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15452 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
15453 %}
15454 ins_pipe( pipe_slow );
15455 %}
15456
15457 // encode char[] to byte[] in ASCII
15458 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
15459 legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
15460 rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
15461 predicate(((EncodeISOArrayNode*)n)->is_ascii());
15462 match(Set result (EncodeISOArray src (Binary dst len)));
15463 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
15464
15465 format %{ "Encode ascii array $src,$dst,$len -> $result // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
15466 ins_encode %{
15467 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
15468 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
15469 $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
15470 %}
15471 ins_pipe( pipe_slow );
15472 %}
15473
15474 //----------Overflow Math Instructions-----------------------------------------
15475
15476 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
15477 %{
15478 match(Set cr (OverflowAddI op1 op2));
15479 effect(DEF cr, USE_KILL op1, USE op2);
15480
15481 format %{ "addl $op1, $op2\t# overflow check int" %}
15482
15483 ins_encode %{
15484 __ addl($op1$$Register, $op2$$Register);
15485 %}
15486 ins_pipe(ialu_reg_reg);
15487 %}
15488
15489 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
15490 %{
15491 match(Set cr (OverflowAddI op1 op2));
15492 effect(DEF cr, USE_KILL op1, USE op2);
15493
15494 format %{ "addl $op1, $op2\t# overflow check int" %}
15495
15496 ins_encode %{
15497 __ addl($op1$$Register, $op2$$constant);
15498 %}
15499 ins_pipe(ialu_reg_reg);
15500 %}
15501
15502 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
15503 %{
15504 match(Set cr (OverflowAddL op1 op2));
15505 effect(DEF cr, USE_KILL op1, USE op2);
15506
15507 format %{ "addq $op1, $op2\t# overflow check long" %}
15508 ins_encode %{
15509 __ addq($op1$$Register, $op2$$Register);
15510 %}
15511 ins_pipe(ialu_reg_reg);
15512 %}
15513
15514 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
15515 %{
15516 match(Set cr (OverflowAddL op1 op2));
15517 effect(DEF cr, USE_KILL op1, USE op2);
15518
15519 format %{ "addq $op1, $op2\t# overflow check long" %}
15520 ins_encode %{
15521 __ addq($op1$$Register, $op2$$constant);
15522 %}
15523 ins_pipe(ialu_reg_reg);
15524 %}
15525
15526 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
15527 %{
15528 match(Set cr (OverflowSubI op1 op2));
15529
15530 format %{ "cmpl $op1, $op2\t# overflow check int" %}
15531 ins_encode %{
15532 __ cmpl($op1$$Register, $op2$$Register);
15533 %}
15534 ins_pipe(ialu_reg_reg);
15535 %}
15536
15537 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
15538 %{
15539 match(Set cr (OverflowSubI op1 op2));
15540
15541 format %{ "cmpl $op1, $op2\t# overflow check int" %}
15542 ins_encode %{
15543 __ cmpl($op1$$Register, $op2$$constant);
15544 %}
15545 ins_pipe(ialu_reg_reg);
15546 %}
15547
15548 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
15549 %{
15550 match(Set cr (OverflowSubL op1 op2));
15551
15552 format %{ "cmpq $op1, $op2\t# overflow check long" %}
15553 ins_encode %{
15554 __ cmpq($op1$$Register, $op2$$Register);
15555 %}
15556 ins_pipe(ialu_reg_reg);
15557 %}
15558
15559 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
15560 %{
15561 match(Set cr (OverflowSubL op1 op2));
15562
15563 format %{ "cmpq $op1, $op2\t# overflow check long" %}
15564 ins_encode %{
15565 __ cmpq($op1$$Register, $op2$$constant);
15566 %}
15567 ins_pipe(ialu_reg_reg);
15568 %}
15569
15570 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
15571 %{
15572 match(Set cr (OverflowSubI zero op2));
15573 effect(DEF cr, USE_KILL op2);
15574
15575 format %{ "negl $op2\t# overflow check int" %}
15576 ins_encode %{
15577 __ negl($op2$$Register);
15578 %}
15579 ins_pipe(ialu_reg_reg);
15580 %}
15581
15582 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
15583 %{
15584 match(Set cr (OverflowSubL zero op2));
15585 effect(DEF cr, USE_KILL op2);
15586
15587 format %{ "negq $op2\t# overflow check long" %}
15588 ins_encode %{
15589 __ negq($op2$$Register);
15590 %}
15591 ins_pipe(ialu_reg_reg);
15592 %}
15593
15594 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
15595 %{
15596 match(Set cr (OverflowMulI op1 op2));
15597 effect(DEF cr, USE_KILL op1, USE op2);
15598
15599 format %{ "imull $op1, $op2\t# overflow check int" %}
15600 ins_encode %{
15601 __ imull($op1$$Register, $op2$$Register);
15602 %}
15603 ins_pipe(ialu_reg_reg_alu0);
15604 %}
15605
15606 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
15607 %{
15608 match(Set cr (OverflowMulI op1 op2));
15609 effect(DEF cr, TEMP tmp, USE op1, USE op2);
15610
15611 format %{ "imull $tmp, $op1, $op2\t# overflow check int" %}
15612 ins_encode %{
15613 __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
15614 %}
15615 ins_pipe(ialu_reg_reg_alu0);
15616 %}
15617
15618 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
15619 %{
15620 match(Set cr (OverflowMulL op1 op2));
15621 effect(DEF cr, USE_KILL op1, USE op2);
15622
15623 format %{ "imulq $op1, $op2\t# overflow check long" %}
15624 ins_encode %{
15625 __ imulq($op1$$Register, $op2$$Register);
15626 %}
15627 ins_pipe(ialu_reg_reg_alu0);
15628 %}
15629
15630 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
15631 %{
15632 match(Set cr (OverflowMulL op1 op2));
15633 effect(DEF cr, TEMP tmp, USE op1, USE op2);
15634
15635 format %{ "imulq $tmp, $op1, $op2\t# overflow check long" %}
15636 ins_encode %{
15637 __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
15638 %}
15639 ins_pipe(ialu_reg_reg_alu0);
15640 %}
15641
15642
15643 //----------Control Flow Instructions------------------------------------------
15644 // Signed compare Instructions
15645
15646 // XXX more variants!!
15647 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
15648 %{
15649 match(Set cr (CmpI op1 op2));
15650 effect(DEF cr, USE op1, USE op2);
15651
15652 format %{ "cmpl $op1, $op2" %}
15653 ins_encode %{
15654 __ cmpl($op1$$Register, $op2$$Register);
15655 %}
15656 ins_pipe(ialu_cr_reg_reg);
15657 %}
15658
15659 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
15660 %{
15661 match(Set cr (CmpI op1 op2));
15662
15663 format %{ "cmpl $op1, $op2" %}
15664 ins_encode %{
15665 __ cmpl($op1$$Register, $op2$$constant);
15666 %}
15667 ins_pipe(ialu_cr_reg_imm);
15668 %}
15669
15670 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
15671 %{
15672 match(Set cr (CmpI op1 (LoadI op2)));
15673
15674 ins_cost(500); // XXX
15675 format %{ "cmpl $op1, $op2" %}
15676 ins_encode %{
15677 __ cmpl($op1$$Register, $op2$$Address);
15678 %}
15679 ins_pipe(ialu_cr_reg_mem);
15680 %}
15681
15682 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
15683 %{
15684 match(Set cr (CmpI src zero));
15685
15686 format %{ "testl $src, $src" %}
15687 ins_encode %{
15688 __ testl($src$$Register, $src$$Register);
15689 %}
15690 ins_pipe(ialu_cr_reg_imm);
15691 %}
15692
15693 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
15694 %{
15695 match(Set cr (CmpI (AndI src con) zero));
15696
15697 format %{ "testl $src, $con" %}
15698 ins_encode %{
15699 __ testl($src$$Register, $con$$constant);
15700 %}
15701 ins_pipe(ialu_cr_reg_imm);
15702 %}
15703
15704 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
15705 %{
15706 match(Set cr (CmpI (AndI src1 src2) zero));
15707
15708 format %{ "testl $src1, $src2" %}
15709 ins_encode %{
15710 __ testl($src1$$Register, $src2$$Register);
15711 %}
15712 ins_pipe(ialu_cr_reg_imm);
15713 %}
15714
15715 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
15716 %{
15717 match(Set cr (CmpI (AndI src (LoadI mem)) zero));
15718
15719 format %{ "testl $src, $mem" %}
15720 ins_encode %{
15721 __ testl($src$$Register, $mem$$Address);
15722 %}
15723 ins_pipe(ialu_cr_reg_mem);
15724 %}
15725
15726 // Unsigned compare Instructions; really, same as signed except they
15727 // produce an rFlagsRegU instead of rFlagsReg.
15728 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
15729 %{
15730 match(Set cr (CmpU op1 op2));
15731
15732 format %{ "cmpl $op1, $op2\t# unsigned" %}
15733 ins_encode %{
15734 __ cmpl($op1$$Register, $op2$$Register);
15735 %}
15736 ins_pipe(ialu_cr_reg_reg);
15737 %}
15738
15739 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
15740 %{
15741 match(Set cr (CmpU op1 op2));
15742
15743 format %{ "cmpl $op1, $op2\t# unsigned" %}
15744 ins_encode %{
15745 __ cmpl($op1$$Register, $op2$$constant);
15746 %}
15747 ins_pipe(ialu_cr_reg_imm);
15748 %}
15749
15750 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
15751 %{
15752 match(Set cr (CmpU op1 (LoadI op2)));
15753
15754 ins_cost(500); // XXX
15755 format %{ "cmpl $op1, $op2\t# unsigned" %}
15756 ins_encode %{
15757 __ cmpl($op1$$Register, $op2$$Address);
15758 %}
15759 ins_pipe(ialu_cr_reg_mem);
15760 %}
15761
15762 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
15763 %{
15764 match(Set cr (CmpU src zero));
15765
15766 format %{ "testl $src, $src\t# unsigned" %}
15767 ins_encode %{
15768 __ testl($src$$Register, $src$$Register);
15769 %}
15770 ins_pipe(ialu_cr_reg_imm);
15771 %}
15772
15773 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
15774 %{
15775 match(Set cr (CmpP op1 op2));
15776
15777 format %{ "cmpq $op1, $op2\t# ptr" %}
15778 ins_encode %{
15779 __ cmpq($op1$$Register, $op2$$Register);
15780 %}
15781 ins_pipe(ialu_cr_reg_reg);
15782 %}
15783
15784 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
15785 %{
15786 match(Set cr (CmpP op1 (LoadP op2)));
15787 predicate(n->in(2)->as_Load()->barrier_data() == 0);
15788
15789 ins_cost(500); // XXX
15790 format %{ "cmpq $op1, $op2\t# ptr" %}
15791 ins_encode %{
15792 __ cmpq($op1$$Register, $op2$$Address);
15793 %}
15794 ins_pipe(ialu_cr_reg_mem);
15795 %}
15796
15797 // XXX this is generalized by compP_rReg_mem???
15798 // Compare raw pointer (used in out-of-heap check).
15799 // Only works because non-oop pointers must be raw pointers
15800 // and raw pointers have no anti-dependencies.
15801 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
15802 %{
15803 predicate(n->in(2)->in(2)->bottom_type()->isa_rawptr() != nullptr &&
15804 n->in(2)->as_Load()->barrier_data() == 0);
15805 match(Set cr (CmpP op1 (LoadP op2)));
15806
15807 format %{ "cmpq $op1, $op2\t# raw ptr" %}
15808 ins_encode %{
15809 __ cmpq($op1$$Register, $op2$$Address);
15810 %}
15811 ins_pipe(ialu_cr_reg_mem);
15812 %}
15813
15814 // This will generate a signed flags result. This should be OK since
15815 // any compare to a zero should be eq/neq.
15816 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
15817 %{
15818 match(Set cr (CmpP src zero));
15819
15820 format %{ "testq $src, $src\t# ptr" %}
15821 ins_encode %{
15822 __ testq($src$$Register, $src$$Register);
15823 %}
15824 ins_pipe(ialu_cr_reg_imm);
15825 %}
15826
15827 // This will generate a signed flags result. This should be OK since
15828 // any compare to a zero should be eq/neq.
15829 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
15830 %{
15831 predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
15832 n->in(1)->as_Load()->barrier_data() == 0);
15833 match(Set cr (CmpP (LoadP op) zero));
15834
15835 ins_cost(500); // XXX
15836 format %{ "testq $op, 0xffffffffffffffff\t# ptr" %}
15837 ins_encode %{
15838 __ testq($op$$Address, 0xFFFFFFFF);
15839 %}
15840 ins_pipe(ialu_cr_reg_imm);
15841 %}
15842
15843 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
15844 %{
15845 predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
15846 n->in(1)->as_Load()->barrier_data() == 0);
15847 match(Set cr (CmpP (LoadP mem) zero));
15848
15849 format %{ "cmpq R12, $mem\t# ptr (R12_heapbase==0)" %}
15850 ins_encode %{
15851 __ cmpq(r12, $mem$$Address);
15852 %}
15853 ins_pipe(ialu_cr_reg_mem);
15854 %}
15855
15856 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
15857 %{
15858 match(Set cr (CmpN op1 op2));
15859
15860 format %{ "cmpl $op1, $op2\t# compressed ptr" %}
15861 ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
15862 ins_pipe(ialu_cr_reg_reg);
15863 %}
15864
15865 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
15866 %{
15867 predicate(n->in(2)->as_Load()->barrier_data() == 0);
15868 match(Set cr (CmpN src (LoadN mem)));
15869
15870 format %{ "cmpl $src, $mem\t# compressed ptr" %}
15871 ins_encode %{
15872 __ cmpl($src$$Register, $mem$$Address);
15873 %}
15874 ins_pipe(ialu_cr_reg_mem);
15875 %}
15876
15877 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
15878 match(Set cr (CmpN op1 op2));
15879
15880 format %{ "cmpl $op1, $op2\t# compressed ptr" %}
15881 ins_encode %{
15882 __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
15883 %}
15884 ins_pipe(ialu_cr_reg_imm);
15885 %}
15886
15887 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
15888 %{
15889 predicate(n->in(2)->as_Load()->barrier_data() == 0);
15890 match(Set cr (CmpN src (LoadN mem)));
15891
15892 format %{ "cmpl $mem, $src\t# compressed ptr" %}
15893 ins_encode %{
15894 __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
15895 %}
15896 ins_pipe(ialu_cr_reg_mem);
15897 %}
15898
15899 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
15900 match(Set cr (CmpN op1 op2));
15901
15902 format %{ "cmpl $op1, $op2\t# compressed klass ptr" %}
15903 ins_encode %{
15904 __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
15905 %}
15906 ins_pipe(ialu_cr_reg_imm);
15907 %}
15908
15909 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
15910 %{
15911 predicate(!UseCompactObjectHeaders);
15912 match(Set cr (CmpN src (LoadNKlass mem)));
15913
15914 format %{ "cmpl $mem, $src\t# compressed klass ptr" %}
15915 ins_encode %{
15916 __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
15917 %}
15918 ins_pipe(ialu_cr_reg_mem);
15919 %}
15920
15921 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
15922 match(Set cr (CmpN src zero));
15923
15924 format %{ "testl $src, $src\t# compressed ptr" %}
15925 ins_encode %{ __ testl($src$$Register, $src$$Register); %}
15926 ins_pipe(ialu_cr_reg_imm);
15927 %}
15928
15929 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
15930 %{
15931 predicate(CompressedOops::base() != nullptr &&
15932 n->in(1)->as_Load()->barrier_data() == 0);
15933 match(Set cr (CmpN (LoadN mem) zero));
15934
15935 ins_cost(500); // XXX
15936 format %{ "testl $mem, 0xffffffff\t# compressed ptr" %}
15937 ins_encode %{
15938 __ cmpl($mem$$Address, (int)0xFFFFFFFF);
15939 %}
15940 ins_pipe(ialu_cr_reg_mem);
15941 %}
15942
15943 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
15944 %{
15945 predicate(CompressedOops::base() == nullptr &&
15946 n->in(1)->as_Load()->barrier_data() == 0);
15947 match(Set cr (CmpN (LoadN mem) zero));
15948
15949 format %{ "cmpl R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
15950 ins_encode %{
15951 __ cmpl(r12, $mem$$Address);
15952 %}
15953 ins_pipe(ialu_cr_reg_mem);
15954 %}
15955
15956 // Yanked all unsigned pointer compare operations.
15957 // Pointer compares are done with CmpP which is already unsigned.
15958
15959 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
15960 %{
15961 match(Set cr (CmpL op1 op2));
15962
15963 format %{ "cmpq $op1, $op2" %}
15964 ins_encode %{
15965 __ cmpq($op1$$Register, $op2$$Register);
15966 %}
15967 ins_pipe(ialu_cr_reg_reg);
15968 %}
15969
15970 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
15971 %{
15972 match(Set cr (CmpL op1 op2));
15973
15974 format %{ "cmpq $op1, $op2" %}
15975 ins_encode %{
15976 __ cmpq($op1$$Register, $op2$$constant);
15977 %}
15978 ins_pipe(ialu_cr_reg_imm);
15979 %}
15980
15981 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
15982 %{
15983 match(Set cr (CmpL op1 (LoadL op2)));
15984
15985 format %{ "cmpq $op1, $op2" %}
15986 ins_encode %{
15987 __ cmpq($op1$$Register, $op2$$Address);
15988 %}
15989 ins_pipe(ialu_cr_reg_mem);
15990 %}
15991
15992 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
15993 %{
15994 match(Set cr (CmpL src zero));
15995
15996 format %{ "testq $src, $src" %}
15997 ins_encode %{
15998 __ testq($src$$Register, $src$$Register);
15999 %}
16000 ins_pipe(ialu_cr_reg_imm);
16001 %}
16002
16003 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
16004 %{
16005 match(Set cr (CmpL (AndL src con) zero));
16006
16007 format %{ "testq $src, $con\t# long" %}
16008 ins_encode %{
16009 __ testq($src$$Register, $con$$constant);
16010 %}
16011 ins_pipe(ialu_cr_reg_imm);
16012 %}
16013
16014 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
16015 %{
16016 match(Set cr (CmpL (AndL src1 src2) zero));
16017
16018 format %{ "testq $src1, $src2\t# long" %}
16019 ins_encode %{
16020 __ testq($src1$$Register, $src2$$Register);
16021 %}
16022 ins_pipe(ialu_cr_reg_imm);
16023 %}
16024
16025 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
16026 %{
16027 match(Set cr (CmpL (AndL src (LoadL mem)) zero));
16028
16029 format %{ "testq $src, $mem" %}
16030 ins_encode %{
16031 __ testq($src$$Register, $mem$$Address);
16032 %}
16033 ins_pipe(ialu_cr_reg_mem);
16034 %}
16035
16036 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
16037 %{
16038 match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
16039
16040 format %{ "testq $src, $mem" %}
16041 ins_encode %{
16042 __ testq($src$$Register, $mem$$Address);
16043 %}
16044 ins_pipe(ialu_cr_reg_mem);
16045 %}
16046
16047 // Manifest a CmpU result in an integer register. Very painful.
16048 // This is the test to avoid.
16049 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
16050 %{
16051 match(Set dst (CmpU3 src1 src2));
16052 effect(KILL flags);
16053
16054 ins_cost(275); // XXX
16055 format %{ "cmpl $src1, $src2\t# CmpL3\n\t"
16056 "movl $dst, -1\n\t"
16057 "jb,u done\n\t"
16058 "setcc $dst \t# emits setne + movzbl or setzune for APX"
16059 "done:" %}
16060 ins_encode %{
16061 Label done;
16062 __ cmpl($src1$$Register, $src2$$Register);
16063 __ movl($dst$$Register, -1);
16064 __ jccb(Assembler::below, done);
16065 __ setcc(Assembler::notZero, $dst$$Register);
16066 __ bind(done);
16067 %}
16068 ins_pipe(pipe_slow);
16069 %}
16070
16071 // Manifest a CmpL result in an integer register. Very painful.
16072 // This is the test to avoid.
16073 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
16074 %{
16075 match(Set dst (CmpL3 src1 src2));
16076 effect(KILL flags);
16077
16078 ins_cost(275); // XXX
16079 format %{ "cmpq $src1, $src2\t# CmpL3\n\t"
16080 "movl $dst, -1\n\t"
16081 "jl,s done\n\t"
16082 "setcc $dst \t# emits setne + movzbl or setzune for APX"
16083 "done:" %}
16084 ins_encode %{
16085 Label done;
16086 __ cmpq($src1$$Register, $src2$$Register);
16087 __ movl($dst$$Register, -1);
16088 __ jccb(Assembler::less, done);
16089 __ setcc(Assembler::notZero, $dst$$Register);
16090 __ bind(done);
16091 %}
16092 ins_pipe(pipe_slow);
16093 %}
16094
16095 // Manifest a CmpUL result in an integer register. Very painful.
16096 // This is the test to avoid.
16097 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
16098 %{
16099 match(Set dst (CmpUL3 src1 src2));
16100 effect(KILL flags);
16101
16102 ins_cost(275); // XXX
16103 format %{ "cmpq $src1, $src2\t# CmpL3\n\t"
16104 "movl $dst, -1\n\t"
16105 "jb,u done\n\t"
16106 "setcc $dst \t# emits setne + movzbl or setzune for APX"
16107 "done:" %}
16108 ins_encode %{
16109 Label done;
16110 __ cmpq($src1$$Register, $src2$$Register);
16111 __ movl($dst$$Register, -1);
16112 __ jccb(Assembler::below, done);
16113 __ setcc(Assembler::notZero, $dst$$Register);
16114 __ bind(done);
16115 %}
16116 ins_pipe(pipe_slow);
16117 %}
16118
16119 // Unsigned long compare Instructions; really, same as signed long except they
16120 // produce an rFlagsRegU instead of rFlagsReg.
16121 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
16122 %{
16123 match(Set cr (CmpUL op1 op2));
16124
16125 format %{ "cmpq $op1, $op2\t# unsigned" %}
16126 ins_encode %{
16127 __ cmpq($op1$$Register, $op2$$Register);
16128 %}
16129 ins_pipe(ialu_cr_reg_reg);
16130 %}
16131
16132 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
16133 %{
16134 match(Set cr (CmpUL op1 op2));
16135
16136 format %{ "cmpq $op1, $op2\t# unsigned" %}
16137 ins_encode %{
16138 __ cmpq($op1$$Register, $op2$$constant);
16139 %}
16140 ins_pipe(ialu_cr_reg_imm);
16141 %}
16142
16143 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
16144 %{
16145 match(Set cr (CmpUL op1 (LoadL op2)));
16146
16147 format %{ "cmpq $op1, $op2\t# unsigned" %}
16148 ins_encode %{
16149 __ cmpq($op1$$Register, $op2$$Address);
16150 %}
16151 ins_pipe(ialu_cr_reg_mem);
16152 %}
16153
16154 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
16155 %{
16156 match(Set cr (CmpUL src zero));
16157
16158 format %{ "testq $src, $src\t# unsigned" %}
16159 ins_encode %{
16160 __ testq($src$$Register, $src$$Register);
16161 %}
16162 ins_pipe(ialu_cr_reg_imm);
16163 %}
16164
16165 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
16166 %{
16167 match(Set cr (CmpI (LoadB mem) imm));
16168
16169 ins_cost(125);
16170 format %{ "cmpb $mem, $imm" %}
16171 ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
16172 ins_pipe(ialu_cr_reg_mem);
16173 %}
16174
16175 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
16176 %{
16177 match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
16178
16179 ins_cost(125);
16180 format %{ "testb $mem, $imm\t# ubyte" %}
16181 ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
16182 ins_pipe(ialu_cr_reg_mem);
16183 %}
16184
16185 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
16186 %{
16187 match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
16188
16189 ins_cost(125);
16190 format %{ "testb $mem, $imm\t# byte" %}
16191 ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
16192 ins_pipe(ialu_cr_reg_mem);
16193 %}
16194
16195 //----------Max and Min--------------------------------------------------------
16196 // Min Instructions
16197
16198 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
16199 %{
16200 predicate(!UseAPX);
16201 effect(USE_DEF dst, USE src, USE cr);
16202
16203 format %{ "cmovlgt $dst, $src\t# min" %}
16204 ins_encode %{
16205 __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
16206 %}
16207 ins_pipe(pipe_cmov_reg);
16208 %}
16209
16210 instruct cmovI_reg_g_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
16211 %{
16212 predicate(UseAPX);
16213 effect(DEF dst, USE src1, USE src2, USE cr);
16214
16215 format %{ "ecmovlgt $dst, $src1, $src2\t# min ndd" %}
16216 ins_encode %{
16217 __ ecmovl(Assembler::greater, $dst$$Register, $src1$$Register, $src2$$Register);
16218 %}
16219 ins_pipe(pipe_cmov_reg);
16220 %}
16221
16222 instruct minI_rReg(rRegI dst, rRegI src)
16223 %{
16224 predicate(!UseAPX);
16225 match(Set dst (MinI dst src));
16226
16227 ins_cost(200);
16228 expand %{
16229 rFlagsReg cr;
16230 compI_rReg(cr, dst, src);
16231 cmovI_reg_g(dst, src, cr);
16232 %}
16233 %}
16234
16235 instruct minI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
16236 %{
16237 predicate(UseAPX);
16238 match(Set dst (MinI src1 src2));
16239 effect(DEF dst, USE src1, USE src2);
16240 flag(PD::Flag_ndd_demotable_opr1);
16241
16242 ins_cost(200);
16243 expand %{
16244 rFlagsReg cr;
16245 compI_rReg(cr, src1, src2);
16246 cmovI_reg_g_ndd(dst, src1, src2, cr);
16247 %}
16248 %}
16249
16250 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
16251 %{
16252 predicate(!UseAPX);
16253 effect(USE_DEF dst, USE src, USE cr);
16254
16255 format %{ "cmovllt $dst, $src\t# max" %}
16256 ins_encode %{
16257 __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
16258 %}
16259 ins_pipe(pipe_cmov_reg);
16260 %}
16261
16262 instruct cmovI_reg_l_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
16263 %{
16264 predicate(UseAPX);
16265 effect(DEF dst, USE src1, USE src2, USE cr);
16266
16267 format %{ "ecmovllt $dst, $src1, $src2\t# max ndd" %}
16268 ins_encode %{
16269 __ ecmovl(Assembler::less, $dst$$Register, $src1$$Register, $src2$$Register);
16270 %}
16271 ins_pipe(pipe_cmov_reg);
16272 %}
16273
16274 instruct maxI_rReg(rRegI dst, rRegI src)
16275 %{
16276 predicate(!UseAPX);
16277 match(Set dst (MaxI dst src));
16278
16279 ins_cost(200);
16280 expand %{
16281 rFlagsReg cr;
16282 compI_rReg(cr, dst, src);
16283 cmovI_reg_l(dst, src, cr);
16284 %}
16285 %}
16286
16287 instruct maxI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
16288 %{
16289 predicate(UseAPX);
16290 match(Set dst (MaxI src1 src2));
16291 effect(DEF dst, USE src1, USE src2);
16292 flag(PD::Flag_ndd_demotable_opr1);
16293
16294 ins_cost(200);
16295 expand %{
16296 rFlagsReg cr;
16297 compI_rReg(cr, src1, src2);
16298 cmovI_reg_l_ndd(dst, src1, src2, cr);
16299 %}
16300 %}
16301
16302 // ============================================================================
16303 // Branch Instructions
16304
16305 // Jump Direct - Label defines a relative address from JMP+1
16306 instruct jmpDir(label labl)
16307 %{
16308 match(Goto);
16309 effect(USE labl);
16310
16311 ins_cost(300);
16312 format %{ "jmp $labl" %}
16313 size(5);
16314 ins_encode %{
16315 Label* L = $labl$$label;
16316 __ jmp(*L, false); // Always long jump
16317 %}
16318 ins_pipe(pipe_jmp);
16319 %}
16320
16321 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16322 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
16323 %{
16324 match(If cop cr);
16325 effect(USE labl);
16326
16327 ins_cost(300);
16328 format %{ "j$cop $labl" %}
16329 size(6);
16330 ins_encode %{
16331 Label* L = $labl$$label;
16332 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16333 %}
16334 ins_pipe(pipe_jcc);
16335 %}
16336
16337 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16338 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
16339 %{
16340 match(CountedLoopEnd cop cr);
16341 effect(USE labl);
16342
16343 ins_cost(300);
16344 format %{ "j$cop $labl\t# loop end" %}
16345 size(6);
16346 ins_encode %{
16347 Label* L = $labl$$label;
16348 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16349 %}
16350 ins_pipe(pipe_jcc);
16351 %}
16352
16353 // Jump Direct Conditional - using unsigned comparison
16354 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
16355 match(If cop cmp);
16356 effect(USE labl);
16357
16358 ins_cost(300);
16359 format %{ "j$cop,u $labl" %}
16360 size(6);
16361 ins_encode %{
16362 Label* L = $labl$$label;
16363 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16364 %}
16365 ins_pipe(pipe_jcc);
16366 %}
16367
16368 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
16369 match(If cop cmp);
16370 effect(USE labl);
16371
16372 ins_cost(200);
16373 format %{ "j$cop,u $labl" %}
16374 size(6);
16375 ins_encode %{
16376 Label* L = $labl$$label;
16377 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16378 %}
16379 ins_pipe(pipe_jcc);
16380 %}
16381
16382 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
16383 match(If cop cmp);
16384 effect(USE labl);
16385
16386 ins_cost(200);
16387 format %{ $$template
16388 if ($cop$$cmpcode == Assembler::notEqual) {
16389 $$emit$$"jp,u $labl\n\t"
16390 $$emit$$"j$cop,u $labl"
16391 } else {
16392 $$emit$$"jp,u done\n\t"
16393 $$emit$$"j$cop,u $labl\n\t"
16394 $$emit$$"done:"
16395 }
16396 %}
16397 ins_encode %{
16398 Label* l = $labl$$label;
16399 if ($cop$$cmpcode == Assembler::notEqual) {
16400 __ jcc(Assembler::parity, *l, false);
16401 __ jcc(Assembler::notEqual, *l, false);
16402 } else if ($cop$$cmpcode == Assembler::equal) {
16403 Label done;
16404 __ jccb(Assembler::parity, done);
16405 __ jcc(Assembler::equal, *l, false);
16406 __ bind(done);
16407 } else {
16408 ShouldNotReachHere();
16409 }
16410 %}
16411 ins_pipe(pipe_jcc);
16412 %}
16413
16414 // Jump Direct Conditional - using signed and unsigned comparison
16415 instruct jmpConUCFE(cmpOpUCFE cop, rFlagsRegUCFE cmp, label labl) %{
16416 match(If cop cmp);
16417 effect(USE labl);
16418
16419 ins_cost(200);
16420 format %{ "j$cop,su $labl" %}
16421 size(6);
16422 ins_encode %{
16423 Label* L = $labl$$label;
16424 __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
16425 %}
16426 ins_pipe(pipe_jcc);
16427 %}
16428
16429 // ============================================================================
16430 // The 2nd slow-half of a subtype check. Scan the subklass's 2ndary
16431 // superklass array for an instance of the superklass. Set a hidden
16432 // internal cache on a hit (cache is checked with exposed code in
16433 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16434 // encoding ALSO sets flags.
16435
16436 instruct partialSubtypeCheck(rdi_RegP result,
16437 rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
16438 rFlagsReg cr)
16439 %{
16440 match(Set result (PartialSubtypeCheck sub super));
16441 predicate(!UseSecondarySupersTable);
16442 effect(KILL rcx, KILL cr);
16443
16444 ins_cost(1100); // slightly larger than the next version
16445 format %{ "movq rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
16446 "movl rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
16447 "addq rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
16448 "repne scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
16449 "jne,s miss\t\t# Missed: rdi not-zero\n\t"
16450 "movq [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
16451 "xorq $result, $result\t\t Hit: rdi zero\n\t"
16452 "miss:\t" %}
16453
16454 ins_encode %{
16455 Label miss;
16456 // NB: Callers may assume that, when $result is a valid register,
16457 // check_klass_subtype_slow_path_linear sets it to a nonzero
16458 // value.
16459 __ check_klass_subtype_slow_path_linear($sub$$Register, $super$$Register,
16460 $rcx$$Register, $result$$Register,
16461 nullptr, &miss,
16462 /*set_cond_codes:*/ true);
16463 __ xorptr($result$$Register, $result$$Register);
16464 __ bind(miss);
16465 %}
16466
16467 ins_pipe(pipe_slow);
16468 %}
16469
16470 // ============================================================================
16471 // Two versions of hashtable-based partialSubtypeCheck, both used when
16472 // we need to search for a super class in the secondary supers array.
16473 // The first is used when we don't know _a priori_ the class being
16474 // searched for. The second, far more common, is used when we do know:
16475 // this is used for instanceof, checkcast, and any case where C2 can
16476 // determine it by constant propagation.
16477
16478 instruct partialSubtypeCheckVarSuper(rsi_RegP sub, rax_RegP super, rdi_RegP result,
16479 rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
16480 rFlagsReg cr)
16481 %{
16482 match(Set result (PartialSubtypeCheck sub super));
16483 predicate(UseSecondarySupersTable);
16484 effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
16485
16486 ins_cost(1000);
16487 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16488
16489 ins_encode %{
16490 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register, $temp1$$Register, $temp2$$Register,
16491 $temp3$$Register, $temp4$$Register, $result$$Register);
16492 %}
16493
16494 ins_pipe(pipe_slow);
16495 %}
16496
16497 instruct partialSubtypeCheckConstSuper(rsi_RegP sub, rax_RegP super_reg, immP super_con, rdi_RegP result,
16498 rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
16499 rFlagsReg cr)
16500 %{
16501 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16502 predicate(UseSecondarySupersTable);
16503 effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
16504
16505 ins_cost(700); // smaller than the next version
16506 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16507
16508 ins_encode %{
16509 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16510 if (InlineSecondarySupersTest) {
16511 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register, $temp1$$Register, $temp2$$Register,
16512 $temp3$$Register, $temp4$$Register, $result$$Register,
16513 super_klass_slot);
16514 } else {
16515 __ call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16516 }
16517 %}
16518
16519 ins_pipe(pipe_slow);
16520 %}
16521
16522 // ============================================================================
16523 // Branch Instructions -- short offset versions
16524 //
16525 // These instructions are used to replace jumps of a long offset (the default
16526 // match) with jumps of a shorter offset. These instructions are all tagged
16527 // with the ins_short_branch attribute, which causes the ADLC to suppress the
16528 // match rules in general matching. Instead, the ADLC generates a conversion
16529 // method in the MachNode which can be used to do in-place replacement of the
16530 // long variant with the shorter variant. The compiler will determine if a
16531 // branch can be taken by the is_short_branch_offset() predicate in the machine
16532 // specific code section of the file.
16533
16534 // Jump Direct - Label defines a relative address from JMP+1
16535 instruct jmpDir_short(label labl) %{
16536 match(Goto);
16537 effect(USE labl);
16538
16539 ins_cost(300);
16540 format %{ "jmp,s $labl" %}
16541 size(2);
16542 ins_encode %{
16543 Label* L = $labl$$label;
16544 __ jmpb(*L);
16545 %}
16546 ins_pipe(pipe_jmp);
16547 ins_short_branch(1);
16548 %}
16549
16550 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16551 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
16552 match(If cop cr);
16553 effect(USE labl);
16554
16555 ins_cost(300);
16556 format %{ "j$cop,s $labl" %}
16557 size(2);
16558 ins_encode %{
16559 Label* L = $labl$$label;
16560 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16561 %}
16562 ins_pipe(pipe_jcc);
16563 ins_short_branch(1);
16564 %}
16565
16566 // Jump Direct Conditional - Label defines a relative address from Jcc+1
16567 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
16568 match(CountedLoopEnd cop cr);
16569 effect(USE labl);
16570
16571 ins_cost(300);
16572 format %{ "j$cop,s $labl\t# loop end" %}
16573 size(2);
16574 ins_encode %{
16575 Label* L = $labl$$label;
16576 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16577 %}
16578 ins_pipe(pipe_jcc);
16579 ins_short_branch(1);
16580 %}
16581
16582 // Jump Direct Conditional - using unsigned comparison
16583 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
16584 match(If cop cmp);
16585 effect(USE labl);
16586
16587 ins_cost(300);
16588 format %{ "j$cop,us $labl" %}
16589 size(2);
16590 ins_encode %{
16591 Label* L = $labl$$label;
16592 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16593 %}
16594 ins_pipe(pipe_jcc);
16595 ins_short_branch(1);
16596 %}
16597
16598 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
16599 match(If cop cmp);
16600 effect(USE labl);
16601
16602 ins_cost(300);
16603 format %{ "j$cop,us $labl" %}
16604 size(2);
16605 ins_encode %{
16606 Label* L = $labl$$label;
16607 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16608 %}
16609 ins_pipe(pipe_jcc);
16610 ins_short_branch(1);
16611 %}
16612
16613 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
16614 match(If cop cmp);
16615 effect(USE labl);
16616
16617 ins_cost(300);
16618 format %{ $$template
16619 if ($cop$$cmpcode == Assembler::notEqual) {
16620 $$emit$$"jp,u,s $labl\n\t"
16621 $$emit$$"j$cop,u,s $labl"
16622 } else {
16623 $$emit$$"jp,u,s done\n\t"
16624 $$emit$$"j$cop,u,s $labl\n\t"
16625 $$emit$$"done:"
16626 }
16627 %}
16628 size(4);
16629 ins_encode %{
16630 Label* l = $labl$$label;
16631 if ($cop$$cmpcode == Assembler::notEqual) {
16632 __ jccb(Assembler::parity, *l);
16633 __ jccb(Assembler::notEqual, *l);
16634 } else if ($cop$$cmpcode == Assembler::equal) {
16635 Label done;
16636 __ jccb(Assembler::parity, done);
16637 __ jccb(Assembler::equal, *l);
16638 __ bind(done);
16639 } else {
16640 ShouldNotReachHere();
16641 }
16642 %}
16643 ins_pipe(pipe_jcc);
16644 ins_short_branch(1);
16645 %}
16646
16647 // Jump Direct Conditional - using signed and unsigned comparison
16648 instruct jmpConUCFE_short(cmpOpUCFE cop, rFlagsRegUCFE cmp, label labl) %{
16649 match(If cop cmp);
16650 effect(USE labl);
16651
16652 ins_cost(300);
16653 format %{ "j$cop,sus $labl" %}
16654 size(2);
16655 ins_encode %{
16656 Label* L = $labl$$label;
16657 __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
16658 %}
16659 ins_pipe(pipe_jcc);
16660 ins_short_branch(1);
16661 %}
16662
16663 // ============================================================================
16664 // inlined locking and unlocking
16665
16666 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
16667 match(Set cr (FastLock object box));
16668 effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
16669 ins_cost(300);
16670 format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
16671 ins_encode %{
16672 __ fast_lock($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
16673 %}
16674 ins_pipe(pipe_slow);
16675 %}
16676
16677 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
16678 match(Set cr (FastUnlock object rax_reg));
16679 effect(TEMP tmp, USE_KILL rax_reg);
16680 ins_cost(300);
16681 format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
16682 ins_encode %{
16683 __ fast_unlock($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
16684 %}
16685 ins_pipe(pipe_slow);
16686 %}
16687
16688
16689 // ============================================================================
16690 // Safepoint Instructions
16691 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
16692 %{
16693 match(SafePoint poll);
16694 effect(KILL cr, USE poll);
16695
16696 format %{ "testl rax, [$poll]\t"
16697 "# Safepoint: poll for GC" %}
16698 ins_cost(125);
16699 ins_encode %{
16700 __ relocate(relocInfo::poll_type);
16701 address pre_pc = __ pc();
16702 __ testl(rax, Address($poll$$Register, 0));
16703 assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
16704 %}
16705 ins_pipe(ialu_reg_mem);
16706 %}
16707
16708 instruct mask_all_evexL(kReg dst, rRegL src) %{
16709 match(Set dst (MaskAll src));
16710 format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
16711 ins_encode %{
16712 int mask_len = Matcher::vector_length(this);
16713 __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
16714 %}
16715 ins_pipe( pipe_slow );
16716 %}
16717
16718 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
16719 predicate(Matcher::vector_length(n) > 32);
16720 match(Set dst (MaskAll src));
16721 effect(TEMP tmp);
16722 format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
16723 ins_encode %{
16724 int mask_len = Matcher::vector_length(this);
16725 __ movslq($tmp$$Register, $src$$Register);
16726 __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
16727 %}
16728 ins_pipe( pipe_slow );
16729 %}
16730
16731 // ============================================================================
16732 // Procedure Call/Return Instructions
16733 // Call Java Static Instruction
16734 // Note: If this code changes, the corresponding ret_addr_offset() and
16735 // compute_padding() functions will have to be adjusted.
16736 instruct CallStaticJavaDirect(method meth) %{
16737 match(CallStaticJava);
16738 effect(USE meth);
16739
16740 ins_cost(300);
16741 format %{ "call,static " %}
16742 opcode(0xE8); /* E8 cd */
16743 ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
16744 ins_pipe(pipe_slow);
16745 ins_alignment(4);
16746 %}
16747
16748 // Call Java Dynamic Instruction
16749 // Note: If this code changes, the corresponding ret_addr_offset() and
16750 // compute_padding() functions will have to be adjusted.
16751 instruct CallDynamicJavaDirect(method meth)
16752 %{
16753 match(CallDynamicJava);
16754 effect(USE meth);
16755
16756 ins_cost(300);
16757 format %{ "movq rax, #Universe::non_oop_word()\n\t"
16758 "call,dynamic " %}
16759 ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
16760 ins_pipe(pipe_slow);
16761 ins_alignment(4);
16762 %}
16763
16764 // Call Runtime Instruction
16765 instruct CallRuntimeDirect(method meth)
16766 %{
16767 match(CallRuntime);
16768 effect(USE meth);
16769
16770 ins_cost(300);
16771 format %{ "call,runtime " %}
16772 ins_encode(clear_avx, Java_To_Runtime(meth));
16773 ins_pipe(pipe_slow);
16774 %}
16775
16776 // Call runtime without safepoint
16777 instruct CallLeafDirect(method meth)
16778 %{
16779 match(CallLeaf);
16780 effect(USE meth);
16781
16782 ins_cost(300);
16783 format %{ "call_leaf,runtime " %}
16784 ins_encode(clear_avx, Java_To_Runtime(meth));
16785 ins_pipe(pipe_slow);
16786 %}
16787
16788 // Call runtime without safepoint and with vector arguments
16789 instruct CallLeafDirectVector(method meth)
16790 %{
16791 match(CallLeafVector);
16792 effect(USE meth);
16793
16794 ins_cost(300);
16795 format %{ "call_leaf,vector " %}
16796 ins_encode(Java_To_Runtime(meth));
16797 ins_pipe(pipe_slow);
16798 %}
16799
16800 // Call runtime without safepoint
16801 instruct CallLeafNoFPDirect(method meth)
16802 %{
16803 match(CallLeafNoFP);
16804 effect(USE meth);
16805
16806 ins_cost(300);
16807 format %{ "call_leaf_nofp,runtime " %}
16808 ins_encode(clear_avx, Java_To_Runtime(meth));
16809 ins_pipe(pipe_slow);
16810 %}
16811
16812 // Return Instruction
16813 // Remove the return address & jump to it.
16814 // Notice: We always emit a nop after a ret to make sure there is room
16815 // for safepoint patching
16816 instruct Ret()
16817 %{
16818 match(Return);
16819
16820 format %{ "ret" %}
16821 ins_encode %{
16822 __ ret(0);
16823 %}
16824 ins_pipe(pipe_jmp);
16825 %}
16826
16827 // Tail Call; Jump from runtime stub to Java code.
16828 // Also known as an 'interprocedural jump'.
16829 // Target of jump will eventually return to caller.
16830 // TailJump below removes the return address.
16831 // Don't use rbp for 'jump_target' because a MachEpilogNode has already been
16832 // emitted just above the TailCall which has reset rbp to the caller state.
16833 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
16834 %{
16835 match(TailCall jump_target method_ptr);
16836
16837 ins_cost(300);
16838 format %{ "jmp $jump_target\t# rbx holds method" %}
16839 ins_encode %{
16840 __ jmp($jump_target$$Register);
16841 %}
16842 ins_pipe(pipe_jmp);
16843 %}
16844
16845 // Tail Jump; remove the return address; jump to target.
16846 // TailCall above leaves the return address around.
16847 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
16848 %{
16849 match(TailJump jump_target ex_oop);
16850
16851 ins_cost(300);
16852 format %{ "popq rdx\t# pop return address\n\t"
16853 "jmp $jump_target" %}
16854 ins_encode %{
16855 __ popq(as_Register(RDX_enc));
16856 __ jmp($jump_target$$Register);
16857 %}
16858 ins_pipe(pipe_jmp);
16859 %}
16860
16861 // Forward exception.
16862 instruct ForwardExceptionjmp()
16863 %{
16864 match(ForwardException);
16865
16866 format %{ "jmp forward_exception_stub" %}
16867 ins_encode %{
16868 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()), noreg);
16869 %}
16870 ins_pipe(pipe_jmp);
16871 %}
16872
16873 // Create exception oop: created by stack-crawling runtime code.
16874 // Created exception is now available to this handler, and is setup
16875 // just prior to jumping to this handler. No code emitted.
16876 instruct CreateException(rax_RegP ex_oop)
16877 %{
16878 match(Set ex_oop (CreateEx));
16879
16880 size(0);
16881 // use the following format syntax
16882 format %{ "# exception oop is in rax; no code emitted" %}
16883 ins_encode();
16884 ins_pipe(empty);
16885 %}
16886
16887 // Rethrow exception:
16888 // The exception oop will come in the first argument position.
16889 // Then JUMP (not call) to the rethrow stub code.
16890 instruct RethrowException()
16891 %{
16892 match(Rethrow);
16893
16894 // use the following format syntax
16895 format %{ "jmp rethrow_stub" %}
16896 ins_encode %{
16897 __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
16898 %}
16899 ins_pipe(pipe_jmp);
16900 %}
16901
16902 // ============================================================================
16903 // This name is KNOWN by the ADLC and cannot be changed.
16904 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16905 // for this guy.
16906 instruct tlsLoadP(r15_RegP dst) %{
16907 match(Set dst (ThreadLocal));
16908 effect(DEF dst);
16909
16910 size(0);
16911 format %{ "# TLS is in R15" %}
16912 ins_encode( /*empty encoding*/ );
16913 ins_pipe(ialu_reg_reg);
16914 %}
16915
16916 instruct addF_reg(regF dst, regF src) %{
16917 predicate(UseAVX == 0);
16918 match(Set dst (AddF dst src));
16919
16920 format %{ "addss $dst, $src" %}
16921 ins_cost(150);
16922 ins_encode %{
16923 __ addss($dst$$XMMRegister, $src$$XMMRegister);
16924 %}
16925 ins_pipe(pipe_slow);
16926 %}
16927
16928 instruct addF_mem(regF dst, memory src) %{
16929 predicate(UseAVX == 0);
16930 match(Set dst (AddF dst (LoadF src)));
16931
16932 format %{ "addss $dst, $src" %}
16933 ins_cost(150);
16934 ins_encode %{
16935 __ addss($dst$$XMMRegister, $src$$Address);
16936 %}
16937 ins_pipe(pipe_slow);
16938 %}
16939
16940 instruct addF_imm(regF dst, immF con) %{
16941 predicate(UseAVX == 0);
16942 match(Set dst (AddF dst con));
16943 format %{ "addss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
16944 ins_cost(150);
16945 ins_encode %{
16946 __ addss($dst$$XMMRegister, $constantaddress($con));
16947 %}
16948 ins_pipe(pipe_slow);
16949 %}
16950
16951 instruct addF_reg_reg(regF dst, regF src1, regF src2) %{
16952 predicate(UseAVX > 0);
16953 match(Set dst (AddF src1 src2));
16954
16955 format %{ "vaddss $dst, $src1, $src2" %}
16956 ins_cost(150);
16957 ins_encode %{
16958 __ vaddss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
16959 %}
16960 ins_pipe(pipe_slow);
16961 %}
16962
16963 instruct addF_reg_mem(regF dst, regF src1, memory src2) %{
16964 predicate(UseAVX > 0);
16965 match(Set dst (AddF src1 (LoadF src2)));
16966
16967 format %{ "vaddss $dst, $src1, $src2" %}
16968 ins_cost(150);
16969 ins_encode %{
16970 __ vaddss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
16971 %}
16972 ins_pipe(pipe_slow);
16973 %}
16974
16975 instruct addF_reg_imm(regF dst, regF src, immF con) %{
16976 predicate(UseAVX > 0);
16977 match(Set dst (AddF src con));
16978
16979 format %{ "vaddss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
16980 ins_cost(150);
16981 ins_encode %{
16982 __ vaddss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
16983 %}
16984 ins_pipe(pipe_slow);
16985 %}
16986
16987 instruct addD_reg(regD dst, regD src) %{
16988 predicate(UseAVX == 0);
16989 match(Set dst (AddD dst src));
16990
16991 format %{ "addsd $dst, $src" %}
16992 ins_cost(150);
16993 ins_encode %{
16994 __ addsd($dst$$XMMRegister, $src$$XMMRegister);
16995 %}
16996 ins_pipe(pipe_slow);
16997 %}
16998
16999 instruct addD_mem(regD dst, memory src) %{
17000 predicate(UseAVX == 0);
17001 match(Set dst (AddD dst (LoadD src)));
17002
17003 format %{ "addsd $dst, $src" %}
17004 ins_cost(150);
17005 ins_encode %{
17006 __ addsd($dst$$XMMRegister, $src$$Address);
17007 %}
17008 ins_pipe(pipe_slow);
17009 %}
17010
17011 instruct addD_imm(regD dst, immD con) %{
17012 predicate(UseAVX == 0);
17013 match(Set dst (AddD dst con));
17014 format %{ "addsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17015 ins_cost(150);
17016 ins_encode %{
17017 __ addsd($dst$$XMMRegister, $constantaddress($con));
17018 %}
17019 ins_pipe(pipe_slow);
17020 %}
17021
17022 instruct addD_reg_reg(regD dst, regD src1, regD src2) %{
17023 predicate(UseAVX > 0);
17024 match(Set dst (AddD src1 src2));
17025
17026 format %{ "vaddsd $dst, $src1, $src2" %}
17027 ins_cost(150);
17028 ins_encode %{
17029 __ vaddsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17030 %}
17031 ins_pipe(pipe_slow);
17032 %}
17033
17034 instruct addD_reg_mem(regD dst, regD src1, memory src2) %{
17035 predicate(UseAVX > 0);
17036 match(Set dst (AddD src1 (LoadD src2)));
17037
17038 format %{ "vaddsd $dst, $src1, $src2" %}
17039 ins_cost(150);
17040 ins_encode %{
17041 __ vaddsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17042 %}
17043 ins_pipe(pipe_slow);
17044 %}
17045
17046 instruct addD_reg_imm(regD dst, regD src, immD con) %{
17047 predicate(UseAVX > 0);
17048 match(Set dst (AddD src con));
17049
17050 format %{ "vaddsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17051 ins_cost(150);
17052 ins_encode %{
17053 __ vaddsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17054 %}
17055 ins_pipe(pipe_slow);
17056 %}
17057
17058 instruct subF_reg(regF dst, regF src) %{
17059 predicate(UseAVX == 0);
17060 match(Set dst (SubF dst src));
17061
17062 format %{ "subss $dst, $src" %}
17063 ins_cost(150);
17064 ins_encode %{
17065 __ subss($dst$$XMMRegister, $src$$XMMRegister);
17066 %}
17067 ins_pipe(pipe_slow);
17068 %}
17069
17070 instruct subF_mem(regF dst, memory src) %{
17071 predicate(UseAVX == 0);
17072 match(Set dst (SubF dst (LoadF src)));
17073
17074 format %{ "subss $dst, $src" %}
17075 ins_cost(150);
17076 ins_encode %{
17077 __ subss($dst$$XMMRegister, $src$$Address);
17078 %}
17079 ins_pipe(pipe_slow);
17080 %}
17081
17082 instruct subF_imm(regF dst, immF con) %{
17083 predicate(UseAVX == 0);
17084 match(Set dst (SubF dst con));
17085 format %{ "subss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
17086 ins_cost(150);
17087 ins_encode %{
17088 __ subss($dst$$XMMRegister, $constantaddress($con));
17089 %}
17090 ins_pipe(pipe_slow);
17091 %}
17092
17093 instruct subF_reg_reg(regF dst, regF src1, regF src2) %{
17094 predicate(UseAVX > 0);
17095 match(Set dst (SubF src1 src2));
17096
17097 format %{ "vsubss $dst, $src1, $src2" %}
17098 ins_cost(150);
17099 ins_encode %{
17100 __ vsubss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17101 %}
17102 ins_pipe(pipe_slow);
17103 %}
17104
17105 instruct subF_reg_mem(regF dst, regF src1, memory src2) %{
17106 predicate(UseAVX > 0);
17107 match(Set dst (SubF src1 (LoadF src2)));
17108
17109 format %{ "vsubss $dst, $src1, $src2" %}
17110 ins_cost(150);
17111 ins_encode %{
17112 __ vsubss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17113 %}
17114 ins_pipe(pipe_slow);
17115 %}
17116
17117 instruct subF_reg_imm(regF dst, regF src, immF con) %{
17118 predicate(UseAVX > 0);
17119 match(Set dst (SubF src con));
17120
17121 format %{ "vsubss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
17122 ins_cost(150);
17123 ins_encode %{
17124 __ vsubss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17125 %}
17126 ins_pipe(pipe_slow);
17127 %}
17128
17129 instruct subD_reg(regD dst, regD src) %{
17130 predicate(UseAVX == 0);
17131 match(Set dst (SubD dst src));
17132
17133 format %{ "subsd $dst, $src" %}
17134 ins_cost(150);
17135 ins_encode %{
17136 __ subsd($dst$$XMMRegister, $src$$XMMRegister);
17137 %}
17138 ins_pipe(pipe_slow);
17139 %}
17140
17141 instruct subD_mem(regD dst, memory src) %{
17142 predicate(UseAVX == 0);
17143 match(Set dst (SubD dst (LoadD src)));
17144
17145 format %{ "subsd $dst, $src" %}
17146 ins_cost(150);
17147 ins_encode %{
17148 __ subsd($dst$$XMMRegister, $src$$Address);
17149 %}
17150 ins_pipe(pipe_slow);
17151 %}
17152
17153 instruct subD_imm(regD dst, immD con) %{
17154 predicate(UseAVX == 0);
17155 match(Set dst (SubD dst con));
17156 format %{ "subsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17157 ins_cost(150);
17158 ins_encode %{
17159 __ subsd($dst$$XMMRegister, $constantaddress($con));
17160 %}
17161 ins_pipe(pipe_slow);
17162 %}
17163
17164 instruct subD_reg_reg(regD dst, regD src1, regD src2) %{
17165 predicate(UseAVX > 0);
17166 match(Set dst (SubD src1 src2));
17167
17168 format %{ "vsubsd $dst, $src1, $src2" %}
17169 ins_cost(150);
17170 ins_encode %{
17171 __ vsubsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17172 %}
17173 ins_pipe(pipe_slow);
17174 %}
17175
17176 instruct subD_reg_mem(regD dst, regD src1, memory src2) %{
17177 predicate(UseAVX > 0);
17178 match(Set dst (SubD src1 (LoadD src2)));
17179
17180 format %{ "vsubsd $dst, $src1, $src2" %}
17181 ins_cost(150);
17182 ins_encode %{
17183 __ vsubsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17184 %}
17185 ins_pipe(pipe_slow);
17186 %}
17187
17188 instruct subD_reg_imm(regD dst, regD src, immD con) %{
17189 predicate(UseAVX > 0);
17190 match(Set dst (SubD src con));
17191
17192 format %{ "vsubsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17193 ins_cost(150);
17194 ins_encode %{
17195 __ vsubsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17196 %}
17197 ins_pipe(pipe_slow);
17198 %}
17199
17200 instruct mulF_reg(regF dst, regF src) %{
17201 predicate(UseAVX == 0);
17202 match(Set dst (MulF dst src));
17203
17204 format %{ "mulss $dst, $src" %}
17205 ins_cost(150);
17206 ins_encode %{
17207 __ mulss($dst$$XMMRegister, $src$$XMMRegister);
17208 %}
17209 ins_pipe(pipe_slow);
17210 %}
17211
17212 instruct mulF_mem(regF dst, memory src) %{
17213 predicate(UseAVX == 0);
17214 match(Set dst (MulF dst (LoadF src)));
17215
17216 format %{ "mulss $dst, $src" %}
17217 ins_cost(150);
17218 ins_encode %{
17219 __ mulss($dst$$XMMRegister, $src$$Address);
17220 %}
17221 ins_pipe(pipe_slow);
17222 %}
17223
17224 instruct mulF_imm(regF dst, immF con) %{
17225 predicate(UseAVX == 0);
17226 match(Set dst (MulF dst con));
17227 format %{ "mulss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
17228 ins_cost(150);
17229 ins_encode %{
17230 __ mulss($dst$$XMMRegister, $constantaddress($con));
17231 %}
17232 ins_pipe(pipe_slow);
17233 %}
17234
17235 instruct mulF_reg_reg(regF dst, regF src1, regF src2) %{
17236 predicate(UseAVX > 0);
17237 match(Set dst (MulF src1 src2));
17238
17239 format %{ "vmulss $dst, $src1, $src2" %}
17240 ins_cost(150);
17241 ins_encode %{
17242 __ vmulss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17243 %}
17244 ins_pipe(pipe_slow);
17245 %}
17246
17247 instruct mulF_reg_mem(regF dst, regF src1, memory src2) %{
17248 predicate(UseAVX > 0);
17249 match(Set dst (MulF src1 (LoadF src2)));
17250
17251 format %{ "vmulss $dst, $src1, $src2" %}
17252 ins_cost(150);
17253 ins_encode %{
17254 __ vmulss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17255 %}
17256 ins_pipe(pipe_slow);
17257 %}
17258
17259 instruct mulF_reg_imm(regF dst, regF src, immF con) %{
17260 predicate(UseAVX > 0);
17261 match(Set dst (MulF src con));
17262
17263 format %{ "vmulss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
17264 ins_cost(150);
17265 ins_encode %{
17266 __ vmulss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17267 %}
17268 ins_pipe(pipe_slow);
17269 %}
17270
17271 instruct mulD_reg(regD dst, regD src) %{
17272 predicate(UseAVX == 0);
17273 match(Set dst (MulD dst src));
17274
17275 format %{ "mulsd $dst, $src" %}
17276 ins_cost(150);
17277 ins_encode %{
17278 __ mulsd($dst$$XMMRegister, $src$$XMMRegister);
17279 %}
17280 ins_pipe(pipe_slow);
17281 %}
17282
17283 instruct mulD_mem(regD dst, memory src) %{
17284 predicate(UseAVX == 0);
17285 match(Set dst (MulD dst (LoadD src)));
17286
17287 format %{ "mulsd $dst, $src" %}
17288 ins_cost(150);
17289 ins_encode %{
17290 __ mulsd($dst$$XMMRegister, $src$$Address);
17291 %}
17292 ins_pipe(pipe_slow);
17293 %}
17294
17295 instruct mulD_imm(regD dst, immD con) %{
17296 predicate(UseAVX == 0);
17297 match(Set dst (MulD dst con));
17298 format %{ "mulsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17299 ins_cost(150);
17300 ins_encode %{
17301 __ mulsd($dst$$XMMRegister, $constantaddress($con));
17302 %}
17303 ins_pipe(pipe_slow);
17304 %}
17305
17306 instruct mulD_reg_reg(regD dst, regD src1, regD src2) %{
17307 predicate(UseAVX > 0);
17308 match(Set dst (MulD src1 src2));
17309
17310 format %{ "vmulsd $dst, $src1, $src2" %}
17311 ins_cost(150);
17312 ins_encode %{
17313 __ vmulsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17314 %}
17315 ins_pipe(pipe_slow);
17316 %}
17317
17318 instruct mulD_reg_mem(regD dst, regD src1, memory src2) %{
17319 predicate(UseAVX > 0);
17320 match(Set dst (MulD src1 (LoadD src2)));
17321
17322 format %{ "vmulsd $dst, $src1, $src2" %}
17323 ins_cost(150);
17324 ins_encode %{
17325 __ vmulsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17326 %}
17327 ins_pipe(pipe_slow);
17328 %}
17329
17330 instruct mulD_reg_imm(regD dst, regD src, immD con) %{
17331 predicate(UseAVX > 0);
17332 match(Set dst (MulD src con));
17333
17334 format %{ "vmulsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17335 ins_cost(150);
17336 ins_encode %{
17337 __ vmulsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17338 %}
17339 ins_pipe(pipe_slow);
17340 %}
17341
17342 instruct divF_reg(regF dst, regF src) %{
17343 predicate(UseAVX == 0);
17344 match(Set dst (DivF dst src));
17345
17346 format %{ "divss $dst, $src" %}
17347 ins_cost(150);
17348 ins_encode %{
17349 __ divss($dst$$XMMRegister, $src$$XMMRegister);
17350 %}
17351 ins_pipe(pipe_slow);
17352 %}
17353
17354 instruct divF_mem(regF dst, memory src) %{
17355 predicate(UseAVX == 0);
17356 match(Set dst (DivF dst (LoadF src)));
17357
17358 format %{ "divss $dst, $src" %}
17359 ins_cost(150);
17360 ins_encode %{
17361 __ divss($dst$$XMMRegister, $src$$Address);
17362 %}
17363 ins_pipe(pipe_slow);
17364 %}
17365
17366 instruct divF_imm(regF dst, immF con) %{
17367 predicate(UseAVX == 0);
17368 match(Set dst (DivF dst con));
17369 format %{ "divss $dst, [$constantaddress]\t# load from constant table: float=$con" %}
17370 ins_cost(150);
17371 ins_encode %{
17372 __ divss($dst$$XMMRegister, $constantaddress($con));
17373 %}
17374 ins_pipe(pipe_slow);
17375 %}
17376
17377 instruct divF_reg_reg(regF dst, regF src1, regF src2) %{
17378 predicate(UseAVX > 0);
17379 match(Set dst (DivF src1 src2));
17380
17381 format %{ "vdivss $dst, $src1, $src2" %}
17382 ins_cost(150);
17383 ins_encode %{
17384 __ vdivss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17385 %}
17386 ins_pipe(pipe_slow);
17387 %}
17388
17389 instruct divF_reg_mem(regF dst, regF src1, memory src2) %{
17390 predicate(UseAVX > 0);
17391 match(Set dst (DivF src1 (LoadF src2)));
17392
17393 format %{ "vdivss $dst, $src1, $src2" %}
17394 ins_cost(150);
17395 ins_encode %{
17396 __ vdivss($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17397 %}
17398 ins_pipe(pipe_slow);
17399 %}
17400
17401 instruct divF_reg_imm(regF dst, regF src, immF con) %{
17402 predicate(UseAVX > 0);
17403 match(Set dst (DivF src con));
17404
17405 format %{ "vdivss $dst, $src, [$constantaddress]\t# load from constant table: float=$con" %}
17406 ins_cost(150);
17407 ins_encode %{
17408 __ vdivss($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17409 %}
17410 ins_pipe(pipe_slow);
17411 %}
17412
17413 instruct divD_reg(regD dst, regD src) %{
17414 predicate(UseAVX == 0);
17415 match(Set dst (DivD dst src));
17416
17417 format %{ "divsd $dst, $src" %}
17418 ins_cost(150);
17419 ins_encode %{
17420 __ divsd($dst$$XMMRegister, $src$$XMMRegister);
17421 %}
17422 ins_pipe(pipe_slow);
17423 %}
17424
17425 instruct divD_mem(regD dst, memory src) %{
17426 predicate(UseAVX == 0);
17427 match(Set dst (DivD dst (LoadD src)));
17428
17429 format %{ "divsd $dst, $src" %}
17430 ins_cost(150);
17431 ins_encode %{
17432 __ divsd($dst$$XMMRegister, $src$$Address);
17433 %}
17434 ins_pipe(pipe_slow);
17435 %}
17436
17437 instruct divD_imm(regD dst, immD con) %{
17438 predicate(UseAVX == 0);
17439 match(Set dst (DivD dst con));
17440 format %{ "divsd $dst, [$constantaddress]\t# load from constant table: double=$con" %}
17441 ins_cost(150);
17442 ins_encode %{
17443 __ divsd($dst$$XMMRegister, $constantaddress($con));
17444 %}
17445 ins_pipe(pipe_slow);
17446 %}
17447
17448 instruct divD_reg_reg(regD dst, regD src1, regD src2) %{
17449 predicate(UseAVX > 0);
17450 match(Set dst (DivD src1 src2));
17451
17452 format %{ "vdivsd $dst, $src1, $src2" %}
17453 ins_cost(150);
17454 ins_encode %{
17455 __ vdivsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
17456 %}
17457 ins_pipe(pipe_slow);
17458 %}
17459
17460 instruct divD_reg_mem(regD dst, regD src1, memory src2) %{
17461 predicate(UseAVX > 0);
17462 match(Set dst (DivD src1 (LoadD src2)));
17463
17464 format %{ "vdivsd $dst, $src1, $src2" %}
17465 ins_cost(150);
17466 ins_encode %{
17467 __ vdivsd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address);
17468 %}
17469 ins_pipe(pipe_slow);
17470 %}
17471
17472 instruct divD_reg_imm(regD dst, regD src, immD con) %{
17473 predicate(UseAVX > 0);
17474 match(Set dst (DivD src con));
17475
17476 format %{ "vdivsd $dst, $src, [$constantaddress]\t# load from constant table: double=$con" %}
17477 ins_cost(150);
17478 ins_encode %{
17479 __ vdivsd($dst$$XMMRegister, $src$$XMMRegister, $constantaddress($con));
17480 %}
17481 ins_pipe(pipe_slow);
17482 %}
17483
17484 instruct absF_reg(regF dst) %{
17485 predicate(UseAVX == 0);
17486 match(Set dst (AbsF dst));
17487 ins_cost(150);
17488 format %{ "andps $dst, [0x7fffffff]\t# abs float by sign masking" %}
17489 ins_encode %{
17490 __ andps($dst$$XMMRegister, ExternalAddress(float_signmask()));
17491 %}
17492 ins_pipe(pipe_slow);
17493 %}
17494
17495 instruct absF_reg_reg(vlRegF dst, vlRegF src) %{
17496 predicate(UseAVX > 0);
17497 match(Set dst (AbsF src));
17498 ins_cost(150);
17499 format %{ "vandps $dst, $src, [0x7fffffff]\t# abs float by sign masking" %}
17500 ins_encode %{
17501 int vlen_enc = Assembler::AVX_128bit;
17502 __ vandps($dst$$XMMRegister, $src$$XMMRegister,
17503 ExternalAddress(float_signmask()), vlen_enc);
17504 %}
17505 ins_pipe(pipe_slow);
17506 %}
17507
17508 instruct absD_reg(regD dst) %{
17509 predicate(UseAVX == 0);
17510 match(Set dst (AbsD dst));
17511 ins_cost(150);
17512 format %{ "andpd $dst, [0x7fffffffffffffff]\t"
17513 "# abs double by sign masking" %}
17514 ins_encode %{
17515 __ andpd($dst$$XMMRegister, ExternalAddress(double_signmask()));
17516 %}
17517 ins_pipe(pipe_slow);
17518 %}
17519
17520 instruct absD_reg_reg(vlRegD dst, vlRegD src) %{
17521 predicate(UseAVX > 0);
17522 match(Set dst (AbsD src));
17523 ins_cost(150);
17524 format %{ "vandpd $dst, $src, [0x7fffffffffffffff]\t"
17525 "# abs double by sign masking" %}
17526 ins_encode %{
17527 int vlen_enc = Assembler::AVX_128bit;
17528 __ vandpd($dst$$XMMRegister, $src$$XMMRegister,
17529 ExternalAddress(double_signmask()), vlen_enc);
17530 %}
17531 ins_pipe(pipe_slow);
17532 %}
17533
17534 instruct negF_reg(regF dst) %{
17535 predicate(UseAVX == 0);
17536 match(Set dst (NegF dst));
17537 ins_cost(150);
17538 format %{ "xorps $dst, [0x80000000]\t# neg float by sign flipping" %}
17539 ins_encode %{
17540 __ xorps($dst$$XMMRegister, ExternalAddress(float_signflip()));
17541 %}
17542 ins_pipe(pipe_slow);
17543 %}
17544
17545 instruct negF_reg_reg(vlRegF dst, vlRegF src) %{
17546 predicate(UseAVX > 0);
17547 match(Set dst (NegF src));
17548 ins_cost(150);
17549 format %{ "vnegatess $dst, $src, [0x80000000]\t# neg float by sign flipping" %}
17550 ins_encode %{
17551 __ vnegatess($dst$$XMMRegister, $src$$XMMRegister,
17552 ExternalAddress(float_signflip()));
17553 %}
17554 ins_pipe(pipe_slow);
17555 %}
17556
17557 instruct negD_reg(regD dst) %{
17558 predicate(UseAVX == 0);
17559 match(Set dst (NegD dst));
17560 ins_cost(150);
17561 format %{ "xorpd $dst, [0x8000000000000000]\t"
17562 "# neg double by sign flipping" %}
17563 ins_encode %{
17564 __ xorpd($dst$$XMMRegister, ExternalAddress(double_signflip()));
17565 %}
17566 ins_pipe(pipe_slow);
17567 %}
17568
17569 instruct negD_reg_reg(vlRegD dst, vlRegD src) %{
17570 predicate(UseAVX > 0);
17571 match(Set dst (NegD src));
17572 ins_cost(150);
17573 format %{ "vnegatesd $dst, $src, [0x8000000000000000]\t"
17574 "# neg double by sign flipping" %}
17575 ins_encode %{
17576 __ vnegatesd($dst$$XMMRegister, $src$$XMMRegister,
17577 ExternalAddress(double_signflip()));
17578 %}
17579 ins_pipe(pipe_slow);
17580 %}
17581
17582 // sqrtss instruction needs destination register to be pre initialized for best performance
17583 // Therefore only the instruct rule where the input is pre-loaded into dst register is defined below
17584 instruct sqrtF_reg(regF dst) %{
17585 match(Set dst (SqrtF dst));
17586 format %{ "sqrtss $dst, $dst" %}
17587 ins_encode %{
17588 __ sqrtss($dst$$XMMRegister, $dst$$XMMRegister);
17589 %}
17590 ins_pipe(pipe_slow);
17591 %}
17592
17593 // sqrtsd instruction needs destination register to be pre initialized for best performance
17594 // Therefore only the instruct rule where the input is pre-loaded into dst register is defined below
17595 instruct sqrtD_reg(regD dst) %{
17596 match(Set dst (SqrtD dst));
17597 format %{ "sqrtsd $dst, $dst" %}
17598 ins_encode %{
17599 __ sqrtsd($dst$$XMMRegister, $dst$$XMMRegister);
17600 %}
17601 ins_pipe(pipe_slow);
17602 %}
17603
17604 instruct convF2HF_reg_reg(rRegI dst, vlRegF src, vlRegF tmp) %{
17605 effect(TEMP tmp);
17606 match(Set dst (ConvF2HF src));
17607 ins_cost(125);
17608 format %{ "vcvtps2ph $dst,$src \t using $tmp as TEMP"%}
17609 ins_encode %{
17610 __ flt_to_flt16($dst$$Register, $src$$XMMRegister, $tmp$$XMMRegister);
17611 %}
17612 ins_pipe( pipe_slow );
17613 %}
17614
17615 instruct convF2HF_mem_reg(memory mem, regF src, kReg ktmp, rRegI rtmp) %{
17616 predicate((UseAVX > 2) && VM_Version::supports_avx512vl());
17617 effect(TEMP ktmp, TEMP rtmp);
17618 match(Set mem (StoreC mem (ConvF2HF src)));
17619 format %{ "evcvtps2ph $mem,$src \t using $ktmp and $rtmp as TEMP" %}
17620 ins_encode %{
17621 __ movl($rtmp$$Register, 0x1);
17622 __ kmovwl($ktmp$$KRegister, $rtmp$$Register);
17623 __ evcvtps2ph($mem$$Address, $ktmp$$KRegister, $src$$XMMRegister, 0x04, Assembler::AVX_128bit);
17624 %}
17625 ins_pipe( pipe_slow );
17626 %}
17627
17628 instruct vconvF2HF(vec dst, vec src) %{
17629 match(Set dst (VectorCastF2HF src));
17630 format %{ "vector_conv_F2HF $dst $src" %}
17631 ins_encode %{
17632 int vlen_enc = vector_length_encoding(this, $src);
17633 __ vcvtps2ph($dst$$XMMRegister, $src$$XMMRegister, 0x04, vlen_enc);
17634 %}
17635 ins_pipe( pipe_slow );
17636 %}
17637
17638 instruct vconvF2HF_mem_reg(memory mem, vec src) %{
17639 predicate(n->as_StoreVector()->memory_size() >= 16);
17640 match(Set mem (StoreVector mem (VectorCastF2HF src)));
17641 format %{ "vcvtps2ph $mem,$src" %}
17642 ins_encode %{
17643 int vlen_enc = vector_length_encoding(this, $src);
17644 __ vcvtps2ph($mem$$Address, $src$$XMMRegister, 0x04, vlen_enc);
17645 %}
17646 ins_pipe( pipe_slow );
17647 %}
17648
17649 instruct convHF2F_reg_reg(vlRegF dst, rRegI src) %{
17650 match(Set dst (ConvHF2F src));
17651 format %{ "vcvtph2ps $dst,$src" %}
17652 ins_encode %{
17653 __ flt16_to_flt($dst$$XMMRegister, $src$$Register);
17654 %}
17655 ins_pipe( pipe_slow );
17656 %}
17657
17658 instruct vconvHF2F_reg_mem(vec dst, memory mem) %{
17659 match(Set dst (VectorCastHF2F (LoadVector mem)));
17660 format %{ "vcvtph2ps $dst,$mem" %}
17661 ins_encode %{
17662 int vlen_enc = vector_length_encoding(this);
17663 __ vcvtph2ps($dst$$XMMRegister, $mem$$Address, vlen_enc);
17664 %}
17665 ins_pipe( pipe_slow );
17666 %}
17667
17668 instruct vconvHF2F(vec dst, vec src) %{
17669 match(Set dst (VectorCastHF2F src));
17670 ins_cost(125);
17671 format %{ "vector_conv_HF2F $dst,$src" %}
17672 ins_encode %{
17673 int vlen_enc = vector_length_encoding(this);
17674 __ vcvtph2ps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
17675 %}
17676 ins_pipe( pipe_slow );
17677 %}
17678
17679 // ---------------------------------------- VectorReinterpret ------------------------------------
17680 instruct reinterpret_mask(kReg dst) %{
17681 predicate(n->bottom_type()->isa_pvectmask() &&
17682 Matcher::vector_length(n) == Matcher::vector_length(n->in(1))); // dst == src
17683 match(Set dst (VectorReinterpret dst));
17684 ins_cost(125);
17685 format %{ "vector_reinterpret $dst\t!" %}
17686 ins_encode %{
17687 // empty
17688 %}
17689 ins_pipe( pipe_slow );
17690 %}
17691
17692 instruct reinterpret_mask_W2B(kReg dst, kReg src, vec xtmp) %{
17693 predicate(UseAVX > 2 && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
17694 n->bottom_type()->isa_pvectmask() &&
17695 n->in(1)->bottom_type()->isa_pvectmask() &&
17696 n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_SHORT &&
17697 n->bottom_type()->is_pvectmask()->element_basic_type() == T_BYTE); // dst == src
17698 match(Set dst (VectorReinterpret src));
17699 effect(TEMP xtmp);
17700 format %{ "vector_mask_reinterpret_W2B $dst $src\t!" %}
17701 ins_encode %{
17702 int src_sz = Matcher::vector_length(this, $src)*type2aelembytes(T_SHORT);
17703 int dst_sz = Matcher::vector_length(this)*type2aelembytes(T_BYTE);
17704 assert(src_sz == dst_sz , "src and dst size mismatch");
17705 int vlen_enc = vector_length_encoding(src_sz);
17706 __ evpmovm2w($xtmp$$XMMRegister, $src$$KRegister, vlen_enc);
17707 __ evpmovb2m($dst$$KRegister, $xtmp$$XMMRegister, vlen_enc);
17708 %}
17709 ins_pipe( pipe_slow );
17710 %}
17711
17712 instruct reinterpret_mask_D2B(kReg dst, kReg src, vec xtmp) %{
17713 predicate(UseAVX > 2 && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
17714 n->bottom_type()->isa_pvectmask() &&
17715 n->in(1)->bottom_type()->isa_pvectmask() &&
17716 (n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_INT ||
17717 n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_FLOAT) &&
17718 n->bottom_type()->is_pvectmask()->element_basic_type() == T_BYTE); // dst == src
17719 match(Set dst (VectorReinterpret src));
17720 effect(TEMP xtmp);
17721 format %{ "vector_mask_reinterpret_D2B $dst $src\t!" %}
17722 ins_encode %{
17723 int src_sz = Matcher::vector_length(this, $src)*type2aelembytes(T_INT);
17724 int dst_sz = Matcher::vector_length(this)*type2aelembytes(T_BYTE);
17725 assert(src_sz == dst_sz , "src and dst size mismatch");
17726 int vlen_enc = vector_length_encoding(src_sz);
17727 __ evpmovm2d($xtmp$$XMMRegister, $src$$KRegister, vlen_enc);
17728 __ evpmovb2m($dst$$KRegister, $xtmp$$XMMRegister, vlen_enc);
17729 %}
17730 ins_pipe( pipe_slow );
17731 %}
17732
17733 instruct reinterpret_mask_Q2B(kReg dst, kReg src, vec xtmp) %{
17734 predicate(UseAVX > 2 && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
17735 n->bottom_type()->isa_pvectmask() &&
17736 n->in(1)->bottom_type()->isa_pvectmask() &&
17737 (n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_LONG ||
17738 n->in(1)->bottom_type()->is_pvectmask()->element_basic_type() == T_DOUBLE) &&
17739 n->bottom_type()->is_pvectmask()->element_basic_type() == T_BYTE); // dst == src
17740 match(Set dst (VectorReinterpret src));
17741 effect(TEMP xtmp);
17742 format %{ "vector_mask_reinterpret_Q2B $dst $src\t!" %}
17743 ins_encode %{
17744 int src_sz = Matcher::vector_length(this, $src)*type2aelembytes(T_LONG);
17745 int dst_sz = Matcher::vector_length(this)*type2aelembytes(T_BYTE);
17746 assert(src_sz == dst_sz , "src and dst size mismatch");
17747 int vlen_enc = vector_length_encoding(src_sz);
17748 __ evpmovm2q($xtmp$$XMMRegister, $src$$KRegister, vlen_enc);
17749 __ evpmovb2m($dst$$KRegister, $xtmp$$XMMRegister, vlen_enc);
17750 %}
17751 ins_pipe( pipe_slow );
17752 %}
17753
17754 instruct reinterpret(vec dst) %{
17755 predicate(!n->bottom_type()->isa_pvectmask() &&
17756 Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1))); // dst == src
17757 match(Set dst (VectorReinterpret dst));
17758 ins_cost(125);
17759 format %{ "vector_reinterpret $dst\t!" %}
17760 ins_encode %{
17761 // empty
17762 %}
17763 ins_pipe( pipe_slow );
17764 %}
17765
17766 instruct reinterpret_expand(vec dst, vec src) %{
17767 predicate(UseAVX == 0 &&
17768 (Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
17769 match(Set dst (VectorReinterpret src));
17770 ins_cost(125);
17771 effect(TEMP dst);
17772 format %{ "vector_reinterpret_expand $dst,$src" %}
17773 ins_encode %{
17774 assert(Matcher::vector_length_in_bytes(this) <= 16, "required");
17775 assert(Matcher::vector_length_in_bytes(this, $src) <= 8, "required");
17776
17777 int src_vlen_in_bytes = Matcher::vector_length_in_bytes(this, $src);
17778 if (src_vlen_in_bytes == 4) {
17779 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_32_bit_mask()), noreg);
17780 } else {
17781 assert(src_vlen_in_bytes == 8, "");
17782 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_64_bit_mask()), noreg);
17783 }
17784 __ pand($dst$$XMMRegister, $src$$XMMRegister);
17785 %}
17786 ins_pipe( pipe_slow );
17787 %}
17788
17789 instruct vreinterpret_expand4(legVec dst, vec src) %{
17790 predicate(UseAVX > 0 &&
17791 !n->bottom_type()->isa_pvectmask() &&
17792 (Matcher::vector_length_in_bytes(n->in(1)) == 4) && // src
17793 (Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
17794 match(Set dst (VectorReinterpret src));
17795 ins_cost(125);
17796 format %{ "vector_reinterpret_expand $dst,$src" %}
17797 ins_encode %{
17798 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_32_bit_mask()), 0, noreg);
17799 %}
17800 ins_pipe( pipe_slow );
17801 %}
17802
17803
17804 instruct vreinterpret_expand(legVec dst, vec src) %{
17805 predicate(UseAVX > 0 &&
17806 !n->bottom_type()->isa_pvectmask() &&
17807 (Matcher::vector_length_in_bytes(n->in(1)) > 4) && // src
17808 (Matcher::vector_length_in_bytes(n->in(1)) < Matcher::vector_length_in_bytes(n))); // src < dst
17809 match(Set dst (VectorReinterpret src));
17810 ins_cost(125);
17811 format %{ "vector_reinterpret_expand $dst,$src\t!" %}
17812 ins_encode %{
17813 switch (Matcher::vector_length_in_bytes(this, $src)) {
17814 case 8: __ movq ($dst$$XMMRegister, $src$$XMMRegister); break;
17815 case 16: __ movdqu ($dst$$XMMRegister, $src$$XMMRegister); break;
17816 case 32: __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister); break;
17817 default: ShouldNotReachHere();
17818 }
17819 %}
17820 ins_pipe( pipe_slow );
17821 %}
17822
17823 instruct reinterpret_shrink(vec dst, legVec src) %{
17824 predicate(!n->bottom_type()->isa_pvectmask() &&
17825 Matcher::vector_length_in_bytes(n->in(1)) > Matcher::vector_length_in_bytes(n)); // src > dst
17826 match(Set dst (VectorReinterpret src));
17827 ins_cost(125);
17828 format %{ "vector_reinterpret_shrink $dst,$src\t!" %}
17829 ins_encode %{
17830 switch (Matcher::vector_length_in_bytes(this)) {
17831 case 4: __ movfltz($dst$$XMMRegister, $src$$XMMRegister); break;
17832 case 8: __ movq ($dst$$XMMRegister, $src$$XMMRegister); break;
17833 case 16: __ movdqu ($dst$$XMMRegister, $src$$XMMRegister); break;
17834 case 32: __ vmovdqu($dst$$XMMRegister, $src$$XMMRegister); break;
17835 default: ShouldNotReachHere();
17836 }
17837 %}
17838 ins_pipe( pipe_slow );
17839 %}
17840
17841 // ----------------------------------------------------------------------------------------------------
17842
17843 instruct roundD_reg(legRegD dst, legRegD src, immU8 rmode) %{
17844 match(Set dst (RoundDoubleMode src rmode));
17845 format %{ "roundsd $dst,$src" %}
17846 ins_cost(150);
17847 ins_encode %{
17848 assert(UseSSE >= 4, "required");
17849 if ((UseAVX == 0) && ($dst$$XMMRegister != $src$$XMMRegister)) {
17850 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
17851 }
17852 __ roundsd($dst$$XMMRegister, $src$$XMMRegister, $rmode$$constant);
17853 %}
17854 ins_pipe(pipe_slow);
17855 %}
17856
17857 instruct roundD_imm(legRegD dst, immD con, immU8 rmode) %{
17858 match(Set dst (RoundDoubleMode con rmode));
17859 format %{ "roundsd $dst,[$constantaddress]\t# load from constant table: double=$con" %}
17860 ins_cost(150);
17861 ins_encode %{
17862 assert(UseSSE >= 4, "required");
17863 __ roundsd($dst$$XMMRegister, $constantaddress($con), $rmode$$constant, noreg);
17864 %}
17865 ins_pipe(pipe_slow);
17866 %}
17867
17868 instruct vroundD_reg(legVec dst, legVec src, immU8 rmode) %{
17869 predicate(Matcher::vector_length(n) < 8);
17870 match(Set dst (RoundDoubleModeV src rmode));
17871 format %{ "vroundpd $dst,$src,$rmode\t! round packedD" %}
17872 ins_encode %{
17873 assert(UseAVX > 0, "required");
17874 int vlen_enc = vector_length_encoding(this);
17875 __ vroundpd($dst$$XMMRegister, $src$$XMMRegister, $rmode$$constant, vlen_enc);
17876 %}
17877 ins_pipe( pipe_slow );
17878 %}
17879
17880 instruct vround8D_reg(vec dst, vec src, immU8 rmode) %{
17881 predicate(Matcher::vector_length(n) == 8);
17882 match(Set dst (RoundDoubleModeV src rmode));
17883 format %{ "vrndscalepd $dst,$src,$rmode\t! round packed8D" %}
17884 ins_encode %{
17885 assert(UseAVX > 2, "required");
17886 __ vrndscalepd($dst$$XMMRegister, $src$$XMMRegister, $rmode$$constant, Assembler::AVX_512bit);
17887 %}
17888 ins_pipe( pipe_slow );
17889 %}
17890
17891 instruct vroundD_mem(legVec dst, memory mem, immU8 rmode) %{
17892 predicate(Matcher::vector_length(n) < 8);
17893 match(Set dst (RoundDoubleModeV (LoadVector mem) rmode));
17894 format %{ "vroundpd $dst, $mem, $rmode\t! round packedD" %}
17895 ins_encode %{
17896 assert(UseAVX > 0, "required");
17897 int vlen_enc = vector_length_encoding(this);
17898 __ vroundpd($dst$$XMMRegister, $mem$$Address, $rmode$$constant, vlen_enc);
17899 %}
17900 ins_pipe( pipe_slow );
17901 %}
17902
17903 instruct vround8D_mem(vec dst, memory mem, immU8 rmode) %{
17904 predicate(Matcher::vector_length(n) == 8);
17905 match(Set dst (RoundDoubleModeV (LoadVector mem) rmode));
17906 format %{ "vrndscalepd $dst,$mem,$rmode\t! round packed8D" %}
17907 ins_encode %{
17908 assert(UseAVX > 2, "required");
17909 __ vrndscalepd($dst$$XMMRegister, $mem$$Address, $rmode$$constant, Assembler::AVX_512bit);
17910 %}
17911 ins_pipe( pipe_slow );
17912 %}
17913
17914 instruct onspinwait() %{
17915 match(OnSpinWait);
17916 ins_cost(200);
17917
17918 format %{
17919 $$template
17920 $$emit$$"pause\t! membar_onspinwait"
17921 %}
17922 ins_encode %{
17923 __ pause();
17924 %}
17925 ins_pipe(pipe_slow);
17926 %}
17927
17928 // a * b + c
17929 instruct fmaD_reg(regD a, regD b, regD c) %{
17930 match(Set c (FmaD c (Binary a b)));
17931 format %{ "fmasd $a,$b,$c\t# $c = $a * $b + $c" %}
17932 ins_cost(150);
17933 ins_encode %{
17934 assert(UseFMA, "Needs FMA instructions support.");
17935 __ fmad($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister);
17936 %}
17937 ins_pipe( pipe_slow );
17938 %}
17939
17940 // a * b + c
17941 instruct fmaF_reg(regF a, regF b, regF c) %{
17942 match(Set c (FmaF c (Binary a b)));
17943 format %{ "fmass $a,$b,$c\t# $c = $a * $b + $c" %}
17944 ins_cost(150);
17945 ins_encode %{
17946 assert(UseFMA, "Needs FMA instructions support.");
17947 __ fmaf($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister);
17948 %}
17949 ins_pipe( pipe_slow );
17950 %}
17951
17952 // ====================VECTOR INSTRUCTIONS=====================================
17953
17954 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
17955 instruct MoveVec2Leg(legVec dst, vec src) %{
17956 match(Set dst src);
17957 format %{ "" %}
17958 ins_encode %{
17959 ShouldNotReachHere();
17960 %}
17961 ins_pipe( fpu_reg_reg );
17962 %}
17963
17964 instruct MoveLeg2Vec(vec dst, legVec src) %{
17965 match(Set dst src);
17966 format %{ "" %}
17967 ins_encode %{
17968 ShouldNotReachHere();
17969 %}
17970 ins_pipe( fpu_reg_reg );
17971 %}
17972
17973 // ============================================================================
17974
17975 // Load vectors generic operand pattern
17976 instruct loadV(vec dst, memory mem) %{
17977 match(Set dst (LoadVector mem));
17978 ins_cost(125);
17979 format %{ "load_vector $dst,$mem" %}
17980 ins_encode %{
17981 BasicType bt = Matcher::vector_element_basic_type(this);
17982 __ load_vector(bt, $dst$$XMMRegister, $mem$$Address, Matcher::vector_length_in_bytes(this));
17983 %}
17984 ins_pipe( pipe_slow );
17985 %}
17986
17987 // Store vectors generic operand pattern.
17988 instruct storeV(memory mem, vec src) %{
17989 match(Set mem (StoreVector mem src));
17990 ins_cost(145);
17991 format %{ "store_vector $mem,$src\n\t" %}
17992 ins_encode %{
17993 switch (Matcher::vector_length_in_bytes(this, $src)) {
17994 case 4: __ movdl ($mem$$Address, $src$$XMMRegister); break;
17995 case 8: __ movq ($mem$$Address, $src$$XMMRegister); break;
17996 case 16: __ movdqu ($mem$$Address, $src$$XMMRegister); break;
17997 case 32: __ vmovdqu ($mem$$Address, $src$$XMMRegister); break;
17998 case 64: __ evmovdqul($mem$$Address, $src$$XMMRegister, Assembler::AVX_512bit); break;
17999 default: ShouldNotReachHere();
18000 }
18001 %}
18002 ins_pipe( pipe_slow );
18003 %}
18004
18005 // ---------------------------------------- Gather ------------------------------------
18006
18007 // Gather BYTE, SHORT, INT, LONG, FLOAT, DOUBLE
18008
18009 instruct gather(legVec dst, memory mem, legVec idx, rRegP tmp, legVec mask) %{
18010 predicate(!VM_Version::supports_avx512vl() && !is_subword_type(Matcher::vector_element_basic_type(n)) &&
18011 Matcher::vector_length_in_bytes(n) <= 32);
18012 match(Set dst (LoadVectorGather mem idx));
18013 effect(TEMP dst, TEMP tmp, TEMP mask);
18014 format %{ "load_vector_gather $dst, $mem, $idx\t! using $tmp and $mask as TEMP" %}
18015 ins_encode %{
18016 int vlen_enc = vector_length_encoding(this);
18017 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18018 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18019 __ vpcmpeqd($mask$$XMMRegister, $mask$$XMMRegister, $mask$$XMMRegister, vlen_enc);
18020 __ lea($tmp$$Register, $mem$$Address);
18021 __ vgather(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx$$XMMRegister, $mask$$XMMRegister, vlen_enc);
18022 %}
18023 ins_pipe( pipe_slow );
18024 %}
18025
18026
18027 instruct evgather(vec dst, memory mem, vec idx, rRegP tmp, kReg ktmp) %{
18028 predicate((VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64) &&
18029 !is_subword_type(Matcher::vector_element_basic_type(n)));
18030 match(Set dst (LoadVectorGather mem idx));
18031 effect(TEMP dst, TEMP tmp, TEMP ktmp);
18032 format %{ "load_vector_gather $dst, $mem, $idx\t! using $tmp and ktmp as TEMP" %}
18033 ins_encode %{
18034 int vlen_enc = vector_length_encoding(this);
18035 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18036 __ kxnorwl($ktmp$$KRegister, $ktmp$$KRegister, $ktmp$$KRegister);
18037 __ lea($tmp$$Register, $mem$$Address);
18038 __ evgather(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $tmp$$Register, $idx$$XMMRegister, vlen_enc);
18039 %}
18040 ins_pipe( pipe_slow );
18041 %}
18042
18043 instruct evgather_masked(vec dst, memory mem, vec idx, kReg mask, kReg ktmp, rRegP tmp) %{
18044 predicate((VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64) &&
18045 !is_subword_type(Matcher::vector_element_basic_type(n)));
18046 match(Set dst (LoadVectorGatherMasked mem (Binary idx mask)));
18047 effect(TEMP_DEF dst, TEMP tmp, TEMP ktmp);
18048 format %{ "load_vector_gather_masked $dst, $mem, $idx, $mask\t! using $tmp and ktmp as TEMP" %}
18049 ins_encode %{
18050 assert(UseAVX > 2, "sanity");
18051 int vlen_enc = vector_length_encoding(this);
18052 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18053 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18054 // Note: Since gather instruction partially updates the opmask register used
18055 // for predication hense moving mask operand to a temporary.
18056 __ kmovwl($ktmp$$KRegister, $mask$$KRegister);
18057 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18058 __ lea($tmp$$Register, $mem$$Address);
18059 __ evgather(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $tmp$$Register, $idx$$XMMRegister, vlen_enc);
18060 %}
18061 ins_pipe( pipe_slow );
18062 %}
18063
18064 instruct vgather_subwordLE8B(vec dst, memory mem, rRegP idx_base, rRegP tmp, rRegI rtmp) %{
18065 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) <= 8);
18066 match(Set dst (LoadVectorGather mem idx_base));
18067 effect(TEMP tmp, TEMP rtmp);
18068 format %{ "vector_gatherLE8 $dst, $mem, $idx_base\t! using $tmp and $rtmp as TEMP" %}
18069 ins_encode %{
18070 int vlen_enc = vector_length_encoding(this);
18071 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18072 __ lea($tmp$$Register, $mem$$Address);
18073 __ vgather8b(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base$$Register, $rtmp$$Register, vlen_enc);
18074 %}
18075 ins_pipe( pipe_slow );
18076 %}
18077
18078 instruct vgather_subwordGT8B(vec dst, memory mem, rRegP idx_base, rRegP tmp, rRegP idx_base_temp,
18079 vec xtmp1, vec xtmp2, vec xtmp3, rRegI rtmp, rRegI length, rFlagsReg cr) %{
18080 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) > 8);
18081 match(Set dst (LoadVectorGather mem idx_base));
18082 effect(TEMP_DEF dst, TEMP tmp, TEMP idx_base_temp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp, TEMP length, KILL cr);
18083 format %{ "vector_gatherGT8 $dst, $mem, $idx_base\t! using $tmp, $idx_base_temp, $xtmp1, $xtmp2, $xtmp3, $rtmp and $length as TEMP" %}
18084 ins_encode %{
18085 int vlen_enc = vector_length_encoding(this);
18086 int vector_len = Matcher::vector_length(this);
18087 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18088 __ lea($tmp$$Register, $mem$$Address);
18089 __ movptr($idx_base_temp$$Register, $idx_base$$Register);
18090 __ vgather_subword(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base_temp$$Register, noreg, $xtmp1$$XMMRegister,
18091 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, noreg, $length$$Register, vector_len, vlen_enc);
18092 %}
18093 ins_pipe( pipe_slow );
18094 %}
18095
18096 instruct vgather_masked_subwordLE8B_avx3(vec dst, memory mem, rRegP idx_base, kReg mask, rRegL mask_idx, rRegP tmp, rRegI rtmp, rRegL rtmp2, rFlagsReg cr) %{
18097 predicate(VM_Version::supports_avx512bw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) <= 8);
18098 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18099 effect(TEMP mask_idx, TEMP tmp, TEMP rtmp, TEMP rtmp2, KILL cr);
18100 format %{ "vector_masked_gatherLE8 $dst, $mem, $idx_base, $mask\t! using $mask_idx, $tmp, $rtmp and $rtmp2 as TEMP" %}
18101 ins_encode %{
18102 int vlen_enc = vector_length_encoding(this);
18103 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18104 __ xorq($mask_idx$$Register, $mask_idx$$Register);
18105 __ lea($tmp$$Register, $mem$$Address);
18106 __ kmovql($rtmp2$$Register, $mask$$KRegister);
18107 __ vgather8b_masked(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base$$Register, $rtmp2$$Register, $mask_idx$$Register, $rtmp$$Register, vlen_enc);
18108 %}
18109 ins_pipe( pipe_slow );
18110 %}
18111
18112 instruct vgather_masked_subwordGT8B_avx3(vec dst, memory mem, rRegP idx_base, kReg mask, rRegP tmp, rRegP idx_base_temp,
18113 vec xtmp1, vec xtmp2, vec xtmp3, rRegI rtmp, rRegL rtmp2, rRegL mask_idx, rRegI length, rFlagsReg cr) %{
18114 predicate(VM_Version::supports_avx512bw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) > 8);
18115 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18116 effect(TEMP_DEF dst, TEMP tmp, TEMP idx_base_temp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp, TEMP rtmp2, TEMP mask_idx, TEMP length, KILL cr);
18117 format %{ "vector_gatherGT8_masked $dst, $mem, $idx_base, $mask\t! using $tmp, $idx_base_temp, $xtmp1, $xtmp2, $xtmp3, $rtmp, $rtmp2, $mask_idx and $length as TEMP" %}
18118 ins_encode %{
18119 int vlen_enc = vector_length_encoding(this);
18120 int vector_len = Matcher::vector_length(this);
18121 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18122 __ xorq($mask_idx$$Register, $mask_idx$$Register);
18123 __ lea($tmp$$Register, $mem$$Address);
18124 __ movptr($idx_base_temp$$Register, $idx_base$$Register);
18125 __ kmovql($rtmp2$$Register, $mask$$KRegister);
18126 __ vgather_subword(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base_temp$$Register, $rtmp2$$Register, $xtmp1$$XMMRegister,
18127 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, $mask_idx$$Register, $length$$Register, vector_len, vlen_enc);
18128 %}
18129 ins_pipe( pipe_slow );
18130 %}
18131
18132 instruct vgather_masked_subwordLE8B_avx2(vec dst, memory mem, rRegP idx_base, vec mask, rRegI mask_idx, rRegP tmp, rRegI rtmp, rRegI rtmp2, rFlagsReg cr) %{
18133 predicate(!VM_Version::supports_avx512vlbw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) <= 8);
18134 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18135 effect(TEMP mask_idx, TEMP tmp, TEMP rtmp, TEMP rtmp2, KILL cr);
18136 format %{ "vector_masked_gatherLE8 $dst, $mem, $idx_base, $mask\t! using $mask_idx, $tmp, $rtmp and $rtmp2 as TEMP" %}
18137 ins_encode %{
18138 int vlen_enc = vector_length_encoding(this);
18139 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18140 __ lea($tmp$$Register, $mem$$Address);
18141 __ vpmovmskb($rtmp2$$Register, $mask$$XMMRegister, vlen_enc);
18142 if (elem_bt == T_SHORT) {
18143 __ movl($mask_idx$$Register, 0x55555555);
18144 __ pextl($rtmp2$$Register, $rtmp2$$Register, $mask_idx$$Register);
18145 }
18146 __ xorl($mask_idx$$Register, $mask_idx$$Register);
18147 __ vgather8b_masked(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base$$Register, $rtmp2$$Register, $mask_idx$$Register, $rtmp$$Register, vlen_enc);
18148 %}
18149 ins_pipe( pipe_slow );
18150 %}
18151
18152 instruct vgather_masked_subwordGT8B_avx2(vec dst, memory mem, rRegP idx_base, vec mask, rRegP tmp, rRegP idx_base_temp,
18153 vec xtmp1, vec xtmp2, vec xtmp3, rRegI rtmp, rRegI rtmp2, rRegI mask_idx, rRegI length, rFlagsReg cr) %{
18154 predicate(!VM_Version::supports_avx512vlbw() && is_subword_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_length_in_bytes(n) > 8);
18155 match(Set dst (LoadVectorGatherMasked mem (Binary idx_base mask)));
18156 effect(TEMP_DEF dst, TEMP tmp, TEMP idx_base_temp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp, TEMP rtmp2, TEMP mask_idx, TEMP length, KILL cr);
18157 format %{ "vector_gatherGT8_masked $dst, $mem, $idx_base, $mask\t! using $tmp, $idx_base_temp, $xtmp1, $xtmp2, $xtmp3, $rtmp, $rtmp2, $mask_idx and $length as TEMP" %}
18158 ins_encode %{
18159 int vlen_enc = vector_length_encoding(this);
18160 int vector_len = Matcher::vector_length(this);
18161 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18162 __ lea($tmp$$Register, $mem$$Address);
18163 __ movptr($idx_base_temp$$Register, $idx_base$$Register);
18164 __ vpmovmskb($rtmp2$$Register, $mask$$XMMRegister, vlen_enc);
18165 if (elem_bt == T_SHORT) {
18166 __ movl($mask_idx$$Register, 0x55555555);
18167 __ pextl($rtmp2$$Register, $rtmp2$$Register, $mask_idx$$Register);
18168 }
18169 __ xorl($mask_idx$$Register, $mask_idx$$Register);
18170 __ vgather_subword(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx_base_temp$$Register, $rtmp2$$Register, $xtmp1$$XMMRegister,
18171 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, $mask_idx$$Register, $length$$Register, vector_len, vlen_enc);
18172 %}
18173 ins_pipe( pipe_slow );
18174 %}
18175
18176 // ====================Scatter=======================================
18177
18178 // Scatter INT, LONG, FLOAT, DOUBLE
18179
18180 instruct scatter(memory mem, vec src, vec idx, rRegP tmp, kReg ktmp) %{
18181 predicate(UseAVX > 2);
18182 match(Set mem (StoreVectorScatter mem (Binary src idx)));
18183 effect(TEMP tmp, TEMP ktmp);
18184 format %{ "store_vector_scatter $mem, $idx, $src\t! using k2 and $tmp as TEMP" %}
18185 ins_encode %{
18186 int vlen_enc = vector_length_encoding(this, $src);
18187 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
18188
18189 assert(Matcher::vector_length_in_bytes(this, $src) >= 16, "sanity");
18190 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18191
18192 __ kmovwl($ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), noreg);
18193 __ lea($tmp$$Register, $mem$$Address);
18194 __ evscatter(elem_bt, $tmp$$Register, $idx$$XMMRegister, $ktmp$$KRegister, $src$$XMMRegister, vlen_enc);
18195 %}
18196 ins_pipe( pipe_slow );
18197 %}
18198
18199 instruct scatter_masked(memory mem, vec src, vec idx, kReg mask, kReg ktmp, rRegP tmp) %{
18200 match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx mask))));
18201 effect(TEMP tmp, TEMP ktmp);
18202 format %{ "store_vector_scatter_masked $mem, $idx, $src, $mask\t!" %}
18203 ins_encode %{
18204 int vlen_enc = vector_length_encoding(this, $src);
18205 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
18206 assert(Matcher::vector_length_in_bytes(this, $src) >= 16, "sanity");
18207 assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
18208 // Note: Since scatter instruction partially updates the opmask register used
18209 // for predication hense moving mask operand to a temporary.
18210 __ kmovwl($ktmp$$KRegister, $mask$$KRegister);
18211 __ lea($tmp$$Register, $mem$$Address);
18212 __ evscatter(elem_bt, $tmp$$Register, $idx$$XMMRegister, $ktmp$$KRegister, $src$$XMMRegister, vlen_enc);
18213 %}
18214 ins_pipe( pipe_slow );
18215 %}
18216
18217 // ====================REPLICATE=======================================
18218
18219 // Replicate byte scalar to be vector
18220 instruct vReplB_reg(vec dst, rRegI src) %{
18221 predicate(Matcher::vector_element_basic_type(n) == T_BYTE);
18222 match(Set dst (Replicate src));
18223 format %{ "replicateB $dst,$src" %}
18224 ins_encode %{
18225 uint vlen = Matcher::vector_length(this);
18226 if (UseAVX >= 2) {
18227 int vlen_enc = vector_length_encoding(this);
18228 if (vlen == 64 || VM_Version::supports_avx512vlbw()) { // AVX512VL for <512bit operands
18229 assert(VM_Version::supports_avx512bw(), "required"); // 512-bit byte vectors assume AVX512BW
18230 __ evpbroadcastb($dst$$XMMRegister, $src$$Register, vlen_enc);
18231 } else {
18232 __ movdl($dst$$XMMRegister, $src$$Register);
18233 __ vpbroadcastb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18234 }
18235 } else {
18236 assert(UseAVX < 2, "");
18237 __ movdl($dst$$XMMRegister, $src$$Register);
18238 __ punpcklbw($dst$$XMMRegister, $dst$$XMMRegister);
18239 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18240 if (vlen >= 16) {
18241 assert(vlen == 16, "");
18242 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18243 }
18244 }
18245 %}
18246 ins_pipe( pipe_slow );
18247 %}
18248
18249 instruct ReplB_mem(vec dst, memory mem) %{
18250 predicate(UseAVX >= 2 && Matcher::vector_element_basic_type(n) == T_BYTE);
18251 match(Set dst (Replicate (LoadB mem)));
18252 format %{ "replicateB $dst,$mem" %}
18253 ins_encode %{
18254 int vlen_enc = vector_length_encoding(this);
18255 __ vpbroadcastb($dst$$XMMRegister, $mem$$Address, vlen_enc);
18256 %}
18257 ins_pipe( pipe_slow );
18258 %}
18259
18260 // ====================ReplicateS=======================================
18261
18262 instruct vReplS_reg(vec dst, rRegI src) %{
18263 predicate(Matcher::vector_element_basic_type(n) == T_SHORT);
18264 match(Set dst (Replicate src));
18265 format %{ "replicateS $dst,$src" %}
18266 ins_encode %{
18267 uint vlen = Matcher::vector_length(this);
18268 int vlen_enc = vector_length_encoding(this);
18269 if (UseAVX >= 2) {
18270 if (vlen == 32 || VM_Version::supports_avx512vlbw()) { // AVX512VL for <512bit operands
18271 assert(VM_Version::supports_avx512bw(), "required"); // 512-bit short vectors assume AVX512BW
18272 __ evpbroadcastw($dst$$XMMRegister, $src$$Register, vlen_enc);
18273 } else {
18274 __ movdl($dst$$XMMRegister, $src$$Register);
18275 __ vpbroadcastw($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18276 }
18277 } else {
18278 assert(UseAVX < 2, "");
18279 __ movdl($dst$$XMMRegister, $src$$Register);
18280 __ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18281 if (vlen >= 8) {
18282 assert(vlen == 8, "");
18283 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18284 }
18285 }
18286 %}
18287 ins_pipe( pipe_slow );
18288 %}
18289
18290 instruct ReplHF_imm(vec dst, immH con, rRegI rtmp) %{
18291 match(Set dst (Replicate con));
18292 effect(TEMP rtmp);
18293 format %{ "replicateHF $dst, $con \t! using $rtmp as TEMP" %}
18294 ins_encode %{
18295 int vlen_enc = vector_length_encoding(this);
18296 BasicType bt = Matcher::vector_element_basic_type(this);
18297 assert(VM_Version::supports_avx512_fp16() && bt == T_SHORT, "");
18298 __ movl($rtmp$$Register, $con$$constant);
18299 __ evpbroadcastw($dst$$XMMRegister, $rtmp$$Register, vlen_enc);
18300 %}
18301 ins_pipe( pipe_slow );
18302 %}
18303
18304 instruct ReplHF_reg(vec dst, regF src, rRegI rtmp) %{
18305 predicate(VM_Version::supports_avx512_fp16() && Matcher::vector_element_basic_type(n) == T_SHORT);
18306 match(Set dst (Replicate src));
18307 effect(TEMP rtmp);
18308 format %{ "replicateHF $dst, $src \t! using $rtmp as TEMP" %}
18309 ins_encode %{
18310 int vlen_enc = vector_length_encoding(this);
18311 __ evmovw($rtmp$$Register, $src$$XMMRegister);
18312 __ evpbroadcastw($dst$$XMMRegister, $rtmp$$Register, vlen_enc);
18313 %}
18314 ins_pipe( pipe_slow );
18315 %}
18316
18317 instruct ReplS_mem(vec dst, memory mem) %{
18318 predicate(UseAVX >= 2 && Matcher::vector_element_basic_type(n) == T_SHORT);
18319 match(Set dst (Replicate (LoadS mem)));
18320 format %{ "replicateS $dst,$mem" %}
18321 ins_encode %{
18322 int vlen_enc = vector_length_encoding(this);
18323 __ vpbroadcastw($dst$$XMMRegister, $mem$$Address, vlen_enc);
18324 %}
18325 ins_pipe( pipe_slow );
18326 %}
18327
18328 // ====================ReplicateI=======================================
18329
18330 instruct ReplI_reg(vec dst, rRegI src) %{
18331 predicate(Matcher::vector_element_basic_type(n) == T_INT);
18332 match(Set dst (Replicate src));
18333 format %{ "replicateI $dst,$src" %}
18334 ins_encode %{
18335 uint vlen = Matcher::vector_length(this);
18336 int vlen_enc = vector_length_encoding(this);
18337 if (vlen == 16 || VM_Version::supports_avx512vl()) { // AVX512VL for <512bit operands
18338 __ evpbroadcastd($dst$$XMMRegister, $src$$Register, vlen_enc);
18339 } else if (VM_Version::supports_avx2()) {
18340 __ movdl($dst$$XMMRegister, $src$$Register);
18341 __ vpbroadcastd($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18342 } else {
18343 __ movdl($dst$$XMMRegister, $src$$Register);
18344 __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18345 }
18346 %}
18347 ins_pipe( pipe_slow );
18348 %}
18349
18350 instruct ReplI_mem(vec dst, memory mem) %{
18351 predicate(Matcher::vector_element_basic_type(n) == T_INT);
18352 match(Set dst (Replicate (LoadI mem)));
18353 format %{ "replicateI $dst,$mem" %}
18354 ins_encode %{
18355 int vlen_enc = vector_length_encoding(this);
18356 if (VM_Version::supports_avx2()) {
18357 __ vpbroadcastd($dst$$XMMRegister, $mem$$Address, vlen_enc);
18358 } else if (VM_Version::supports_avx()) {
18359 __ vbroadcastss($dst$$XMMRegister, $mem$$Address, vlen_enc);
18360 } else {
18361 __ movdl($dst$$XMMRegister, $mem$$Address);
18362 __ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
18363 }
18364 %}
18365 ins_pipe( pipe_slow );
18366 %}
18367
18368 instruct ReplI_imm(vec dst, immI con) %{
18369 predicate(Matcher::is_non_long_integral_vector(n));
18370 match(Set dst (Replicate con));
18371 format %{ "replicateI $dst,$con" %}
18372 ins_encode %{
18373 InternalAddress addr = $constantaddress(vreplicate_imm(Matcher::vector_element_basic_type(this), $con$$constant,
18374 (VM_Version::supports_sse3() ? (VM_Version::supports_avx() ? 4 : 8) : 16) /
18375 type2aelembytes(Matcher::vector_element_basic_type(this))));
18376 BasicType bt = Matcher::vector_element_basic_type(this);
18377 int vlen = Matcher::vector_length_in_bytes(this);
18378 __ load_constant_vector(bt, $dst$$XMMRegister, addr, vlen);
18379 %}
18380 ins_pipe( pipe_slow );
18381 %}
18382
18383 // Replicate scalar zero to be vector
18384 instruct ReplI_zero(vec dst, immI_0 zero) %{
18385 predicate(Matcher::is_non_long_integral_vector(n));
18386 match(Set dst (Replicate zero));
18387 format %{ "replicateI $dst,$zero" %}
18388 ins_encode %{
18389 int vlen_enc = vector_length_encoding(this);
18390 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vl()) {
18391 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18392 } else {
18393 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
18394 }
18395 %}
18396 ins_pipe( fpu_reg_reg );
18397 %}
18398
18399 instruct ReplI_M1(vec dst, immI_M1 con) %{
18400 predicate(Matcher::is_non_long_integral_vector(n));
18401 match(Set dst (Replicate con));
18402 format %{ "vallones $dst" %}
18403 ins_encode %{
18404 int vector_len = vector_length_encoding(this);
18405 __ vallones($dst$$XMMRegister, vector_len);
18406 %}
18407 ins_pipe( pipe_slow );
18408 %}
18409
18410 // ====================ReplicateL=======================================
18411
18412 // Replicate long (8 byte) scalar to be vector
18413 instruct ReplL_reg(vec dst, rRegL src) %{
18414 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18415 match(Set dst (Replicate src));
18416 format %{ "replicateL $dst,$src" %}
18417 ins_encode %{
18418 int vlen = Matcher::vector_length(this);
18419 int vlen_enc = vector_length_encoding(this);
18420 if (vlen == 8 || VM_Version::supports_avx512vl()) { // AVX512VL for <512bit operands
18421 __ evpbroadcastq($dst$$XMMRegister, $src$$Register, vlen_enc);
18422 } else if (VM_Version::supports_avx2()) {
18423 __ movdq($dst$$XMMRegister, $src$$Register);
18424 __ vpbroadcastq($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18425 } else {
18426 __ movdq($dst$$XMMRegister, $src$$Register);
18427 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18428 }
18429 %}
18430 ins_pipe( pipe_slow );
18431 %}
18432
18433 instruct ReplL_mem(vec dst, memory mem) %{
18434 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18435 match(Set dst (Replicate (LoadL mem)));
18436 format %{ "replicateL $dst,$mem" %}
18437 ins_encode %{
18438 int vlen_enc = vector_length_encoding(this);
18439 if (VM_Version::supports_avx2()) {
18440 __ vpbroadcastq($dst$$XMMRegister, $mem$$Address, vlen_enc);
18441 } else if (VM_Version::supports_sse3()) {
18442 __ movddup($dst$$XMMRegister, $mem$$Address);
18443 } else {
18444 __ movq($dst$$XMMRegister, $mem$$Address);
18445 __ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
18446 }
18447 %}
18448 ins_pipe( pipe_slow );
18449 %}
18450
18451 // Replicate long (8 byte) scalar immediate to be vector by loading from const table.
18452 instruct ReplL_imm(vec dst, immL con) %{
18453 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18454 match(Set dst (Replicate con));
18455 format %{ "replicateL $dst,$con" %}
18456 ins_encode %{
18457 InternalAddress addr = $constantaddress(vreplicate_imm(T_LONG, $con$$constant, VM_Version::supports_sse3() ? 1 : 2));
18458 int vlen = Matcher::vector_length_in_bytes(this);
18459 __ load_constant_vector(T_LONG, $dst$$XMMRegister, addr, vlen);
18460 %}
18461 ins_pipe( pipe_slow );
18462 %}
18463
18464 instruct ReplL_zero(vec dst, immL0 zero) %{
18465 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18466 match(Set dst (Replicate zero));
18467 format %{ "replicateL $dst,$zero" %}
18468 ins_encode %{
18469 int vlen_enc = vector_length_encoding(this);
18470 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vl()) {
18471 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18472 } else {
18473 __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
18474 }
18475 %}
18476 ins_pipe( fpu_reg_reg );
18477 %}
18478
18479 instruct ReplL_M1(vec dst, immL_M1 con) %{
18480 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
18481 match(Set dst (Replicate con));
18482 format %{ "vallones $dst" %}
18483 ins_encode %{
18484 int vector_len = vector_length_encoding(this);
18485 __ vallones($dst$$XMMRegister, vector_len);
18486 %}
18487 ins_pipe( pipe_slow );
18488 %}
18489
18490 // ====================ReplicateF=======================================
18491
18492 instruct vReplF_reg(vec dst, vlRegF src) %{
18493 predicate(UseAVX > 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
18494 match(Set dst (Replicate src));
18495 format %{ "replicateF $dst,$src" %}
18496 ins_encode %{
18497 uint vlen = Matcher::vector_length(this);
18498 int vlen_enc = vector_length_encoding(this);
18499 if (vlen <= 4) {
18500 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 0x00, Assembler::AVX_128bit);
18501 } else if (VM_Version::supports_avx2()) {
18502 __ vbroadcastss($dst$$XMMRegister, $src$$XMMRegister, vlen_enc); // reg-to-reg variant requires AVX2
18503 } else {
18504 assert(vlen == 8, "sanity");
18505 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 0x00, Assembler::AVX_128bit);
18506 __ vinsertf128_high($dst$$XMMRegister, $dst$$XMMRegister);
18507 }
18508 %}
18509 ins_pipe( pipe_slow );
18510 %}
18511
18512 instruct ReplF_reg(vec dst, vlRegF src) %{
18513 predicate(UseAVX == 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
18514 match(Set dst (Replicate src));
18515 format %{ "replicateF $dst,$src" %}
18516 ins_encode %{
18517 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x00);
18518 %}
18519 ins_pipe( pipe_slow );
18520 %}
18521
18522 instruct ReplF_mem(vec dst, memory mem) %{
18523 predicate(UseAVX > 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
18524 match(Set dst (Replicate (LoadF mem)));
18525 format %{ "replicateF $dst,$mem" %}
18526 ins_encode %{
18527 int vlen_enc = vector_length_encoding(this);
18528 __ vbroadcastss($dst$$XMMRegister, $mem$$Address, vlen_enc);
18529 %}
18530 ins_pipe( pipe_slow );
18531 %}
18532
18533 // Replicate float scalar immediate to be vector by loading from const table.
18534 instruct ReplF_imm(vec dst, immF con) %{
18535 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
18536 match(Set dst (Replicate con));
18537 format %{ "replicateF $dst,$con" %}
18538 ins_encode %{
18539 InternalAddress addr = $constantaddress(vreplicate_imm(T_FLOAT, $con$$constant,
18540 VM_Version::supports_sse3() ? (VM_Version::supports_avx() ? 1 : 2) : 4));
18541 int vlen = Matcher::vector_length_in_bytes(this);
18542 __ load_constant_vector(T_FLOAT, $dst$$XMMRegister, addr, vlen);
18543 %}
18544 ins_pipe( pipe_slow );
18545 %}
18546
18547 instruct ReplF_zero(vec dst, immF0 zero) %{
18548 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
18549 match(Set dst (Replicate zero));
18550 format %{ "replicateF $dst,$zero" %}
18551 ins_encode %{
18552 int vlen_enc = vector_length_encoding(this);
18553 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vldq()) {
18554 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18555 } else {
18556 __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
18557 }
18558 %}
18559 ins_pipe( fpu_reg_reg );
18560 %}
18561
18562 // ====================ReplicateD=======================================
18563
18564 // Replicate double (8 bytes) scalar to be vector
18565 instruct vReplD_reg(vec dst, vlRegD src) %{
18566 predicate(UseSSE >= 3 && Matcher::vector_element_basic_type(n) == T_DOUBLE);
18567 match(Set dst (Replicate src));
18568 format %{ "replicateD $dst,$src" %}
18569 ins_encode %{
18570 uint vlen = Matcher::vector_length(this);
18571 int vlen_enc = vector_length_encoding(this);
18572 if (vlen <= 2) {
18573 __ movddup($dst$$XMMRegister, $src$$XMMRegister);
18574 } else if (VM_Version::supports_avx2()) {
18575 __ vbroadcastsd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc); // reg-to-reg variant requires AVX2
18576 } else {
18577 assert(vlen == 4, "sanity");
18578 __ movddup($dst$$XMMRegister, $src$$XMMRegister);
18579 __ vinsertf128_high($dst$$XMMRegister, $dst$$XMMRegister);
18580 }
18581 %}
18582 ins_pipe( pipe_slow );
18583 %}
18584
18585 instruct ReplD_reg(vec dst, vlRegD src) %{
18586 predicate(UseSSE < 3 && Matcher::vector_element_basic_type(n) == T_DOUBLE);
18587 match(Set dst (Replicate src));
18588 format %{ "replicateD $dst,$src" %}
18589 ins_encode %{
18590 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x44);
18591 %}
18592 ins_pipe( pipe_slow );
18593 %}
18594
18595 instruct ReplD_mem(vec dst, memory mem) %{
18596 predicate(UseSSE >= 3 && Matcher::vector_element_basic_type(n) == T_DOUBLE);
18597 match(Set dst (Replicate (LoadD mem)));
18598 format %{ "replicateD $dst,$mem" %}
18599 ins_encode %{
18600 if (Matcher::vector_length(this) >= 4) {
18601 int vlen_enc = vector_length_encoding(this);
18602 __ vbroadcastsd($dst$$XMMRegister, $mem$$Address, vlen_enc);
18603 } else {
18604 __ movddup($dst$$XMMRegister, $mem$$Address);
18605 }
18606 %}
18607 ins_pipe( pipe_slow );
18608 %}
18609
18610 // Replicate double (8 byte) scalar immediate to be vector by loading from const table.
18611 instruct ReplD_imm(vec dst, immD con) %{
18612 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
18613 match(Set dst (Replicate con));
18614 format %{ "replicateD $dst,$con" %}
18615 ins_encode %{
18616 InternalAddress addr = $constantaddress(vreplicate_imm(T_DOUBLE, $con$$constant, VM_Version::supports_sse3() ? 1 : 2));
18617 int vlen = Matcher::vector_length_in_bytes(this);
18618 __ load_constant_vector(T_DOUBLE, $dst$$XMMRegister, addr, vlen);
18619 %}
18620 ins_pipe( pipe_slow );
18621 %}
18622
18623 instruct ReplD_zero(vec dst, immD0 zero) %{
18624 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
18625 match(Set dst (Replicate zero));
18626 format %{ "replicateD $dst,$zero" %}
18627 ins_encode %{
18628 int vlen_enc = vector_length_encoding(this);
18629 if (VM_Version::supports_evex() && !VM_Version::supports_avx512vldq()) {
18630 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
18631 } else {
18632 __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
18633 }
18634 %}
18635 ins_pipe( fpu_reg_reg );
18636 %}
18637
18638 // ====================VECTOR INSERT=======================================
18639
18640 instruct insert(vec dst, rRegI val, immU8 idx) %{
18641 predicate(Matcher::vector_length_in_bytes(n) < 32);
18642 match(Set dst (VectorInsert (Binary dst val) idx));
18643 format %{ "vector_insert $dst,$val,$idx" %}
18644 ins_encode %{
18645 assert(UseSSE >= 4, "required");
18646 assert(Matcher::vector_length_in_bytes(this) >= 8, "required");
18647
18648 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18649
18650 assert(is_integral_type(elem_bt), "");
18651 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18652
18653 __ insert(elem_bt, $dst$$XMMRegister, $val$$Register, $idx$$constant);
18654 %}
18655 ins_pipe( pipe_slow );
18656 %}
18657
18658 instruct insert32(vec dst, vec src, rRegI val, immU8 idx, vec vtmp) %{
18659 predicate(Matcher::vector_length_in_bytes(n) == 32);
18660 match(Set dst (VectorInsert (Binary src val) idx));
18661 effect(TEMP vtmp);
18662 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18663 ins_encode %{
18664 int vlen_enc = Assembler::AVX_256bit;
18665 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18666 int elem_per_lane = 16/type2aelembytes(elem_bt);
18667 int log2epr = log2(elem_per_lane);
18668
18669 assert(is_integral_type(elem_bt), "sanity");
18670 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18671
18672 uint x_idx = $idx$$constant & right_n_bits(log2epr);
18673 uint y_idx = ($idx$$constant >> log2epr) & 1;
18674 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18675 __ vinsert(elem_bt, $vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
18676 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18677 %}
18678 ins_pipe( pipe_slow );
18679 %}
18680
18681 instruct insert64(vec dst, vec src, rRegI val, immU8 idx, legVec vtmp) %{
18682 predicate(Matcher::vector_length_in_bytes(n) == 64);
18683 match(Set dst (VectorInsert (Binary src val) idx));
18684 effect(TEMP vtmp);
18685 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18686 ins_encode %{
18687 assert(UseAVX > 2, "sanity");
18688
18689 BasicType elem_bt = Matcher::vector_element_basic_type(this);
18690 int elem_per_lane = 16/type2aelembytes(elem_bt);
18691 int log2epr = log2(elem_per_lane);
18692
18693 assert(is_integral_type(elem_bt), "");
18694 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18695
18696 uint x_idx = $idx$$constant & right_n_bits(log2epr);
18697 uint y_idx = ($idx$$constant >> log2epr) & 3;
18698 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18699 __ vinsert(elem_bt, $vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
18700 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18701 %}
18702 ins_pipe( pipe_slow );
18703 %}
18704
18705 instruct insert2L(vec dst, rRegL val, immU8 idx) %{
18706 predicate(Matcher::vector_length(n) == 2);
18707 match(Set dst (VectorInsert (Binary dst val) idx));
18708 format %{ "vector_insert $dst,$val,$idx" %}
18709 ins_encode %{
18710 assert(UseSSE >= 4, "required");
18711 assert(Matcher::vector_element_basic_type(this) == T_LONG, "");
18712 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18713
18714 __ pinsrq($dst$$XMMRegister, $val$$Register, $idx$$constant);
18715 %}
18716 ins_pipe( pipe_slow );
18717 %}
18718
18719 instruct insert4L(vec dst, vec src, rRegL val, immU8 idx, vec vtmp) %{
18720 predicate(Matcher::vector_length(n) == 4);
18721 match(Set dst (VectorInsert (Binary src val) idx));
18722 effect(TEMP vtmp);
18723 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18724 ins_encode %{
18725 assert(Matcher::vector_element_basic_type(this) == T_LONG, "");
18726 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18727
18728 uint x_idx = $idx$$constant & right_n_bits(1);
18729 uint y_idx = ($idx$$constant >> 1) & 1;
18730 int vlen_enc = Assembler::AVX_256bit;
18731 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18732 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
18733 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18734 %}
18735 ins_pipe( pipe_slow );
18736 %}
18737
18738 instruct insert8L(vec dst, vec src, rRegL val, immU8 idx, legVec vtmp) %{
18739 predicate(Matcher::vector_length(n) == 8);
18740 match(Set dst (VectorInsert (Binary src val) idx));
18741 effect(TEMP vtmp);
18742 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18743 ins_encode %{
18744 assert(Matcher::vector_element_basic_type(this) == T_LONG, "sanity");
18745 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18746
18747 uint x_idx = $idx$$constant & right_n_bits(1);
18748 uint y_idx = ($idx$$constant >> 1) & 3;
18749 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18750 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$Register, x_idx);
18751 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18752 %}
18753 ins_pipe( pipe_slow );
18754 %}
18755
18756 instruct insertF(vec dst, regF val, immU8 idx) %{
18757 predicate(Matcher::vector_length(n) < 8);
18758 match(Set dst (VectorInsert (Binary dst val) idx));
18759 format %{ "vector_insert $dst,$val,$idx" %}
18760 ins_encode %{
18761 assert(UseSSE >= 4, "sanity");
18762
18763 assert(Matcher::vector_element_basic_type(this) == T_FLOAT, "sanity");
18764 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18765
18766 uint x_idx = $idx$$constant & right_n_bits(2);
18767 __ insertps($dst$$XMMRegister, $val$$XMMRegister, x_idx << 4);
18768 %}
18769 ins_pipe( pipe_slow );
18770 %}
18771
18772 instruct vinsertF(vec dst, vec src, regF val, immU8 idx, vec vtmp) %{
18773 predicate(Matcher::vector_length(n) >= 8);
18774 match(Set dst (VectorInsert (Binary src val) idx));
18775 effect(TEMP vtmp);
18776 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18777 ins_encode %{
18778 assert(Matcher::vector_element_basic_type(this) == T_FLOAT, "sanity");
18779 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18780
18781 int vlen = Matcher::vector_length(this);
18782 uint x_idx = $idx$$constant & right_n_bits(2);
18783 if (vlen == 8) {
18784 uint y_idx = ($idx$$constant >> 2) & 1;
18785 int vlen_enc = Assembler::AVX_256bit;
18786 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18787 __ vinsertps($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$XMMRegister, x_idx << 4);
18788 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18789 } else {
18790 assert(vlen == 16, "sanity");
18791 uint y_idx = ($idx$$constant >> 2) & 3;
18792 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18793 __ vinsertps($vtmp$$XMMRegister, $vtmp$$XMMRegister, $val$$XMMRegister, x_idx << 4);
18794 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18795 }
18796 %}
18797 ins_pipe( pipe_slow );
18798 %}
18799
18800 instruct insert2D(vec dst, regD val, immU8 idx, rRegL tmp) %{
18801 predicate(Matcher::vector_length(n) == 2);
18802 match(Set dst (VectorInsert (Binary dst val) idx));
18803 effect(TEMP tmp);
18804 format %{ "vector_insert $dst,$val,$idx\t!using $tmp as TEMP" %}
18805 ins_encode %{
18806 assert(UseSSE >= 4, "sanity");
18807 assert(Matcher::vector_element_basic_type(this) == T_DOUBLE, "sanity");
18808 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18809
18810 __ movq($tmp$$Register, $val$$XMMRegister);
18811 __ pinsrq($dst$$XMMRegister, $tmp$$Register, $idx$$constant);
18812 %}
18813 ins_pipe( pipe_slow );
18814 %}
18815
18816 instruct insert4D(vec dst, vec src, regD val, immU8 idx, rRegL tmp, vec vtmp) %{
18817 predicate(Matcher::vector_length(n) == 4);
18818 match(Set dst (VectorInsert (Binary src val) idx));
18819 effect(TEMP vtmp, TEMP tmp);
18820 format %{ "vector_insert $dst,$src,$val,$idx\t!using $tmp, $vtmp as TEMP" %}
18821 ins_encode %{
18822 assert(Matcher::vector_element_basic_type(this) == T_DOUBLE, "sanity");
18823 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18824
18825 uint x_idx = $idx$$constant & right_n_bits(1);
18826 uint y_idx = ($idx$$constant >> 1) & 1;
18827 int vlen_enc = Assembler::AVX_256bit;
18828 __ movq($tmp$$Register, $val$$XMMRegister);
18829 __ vextracti128($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18830 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $tmp$$Register, x_idx);
18831 __ vinserti128($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18832 %}
18833 ins_pipe( pipe_slow );
18834 %}
18835
18836 instruct insert8D(vec dst, vec src, regD val, immI idx, rRegL tmp, legVec vtmp) %{
18837 predicate(Matcher::vector_length(n) == 8);
18838 match(Set dst (VectorInsert (Binary src val) idx));
18839 effect(TEMP tmp, TEMP vtmp);
18840 format %{ "vector_insert $dst,$src,$val,$idx\t!using $vtmp as TEMP" %}
18841 ins_encode %{
18842 assert(Matcher::vector_element_basic_type(this) == T_DOUBLE, "sanity");
18843 assert($idx$$constant < (int)Matcher::vector_length(this), "out of bounds");
18844
18845 uint x_idx = $idx$$constant & right_n_bits(1);
18846 uint y_idx = ($idx$$constant >> 1) & 3;
18847 __ movq($tmp$$Register, $val$$XMMRegister);
18848 __ vextracti32x4($vtmp$$XMMRegister, $src$$XMMRegister, y_idx);
18849 __ vpinsrq($vtmp$$XMMRegister, $vtmp$$XMMRegister, $tmp$$Register, x_idx);
18850 __ vinserti32x4($dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister, y_idx);
18851 %}
18852 ins_pipe( pipe_slow );
18853 %}
18854
18855 // ====================REDUCTION ARITHMETIC=======================================
18856
18857 // =======================Int Reduction==========================================
18858
18859 instruct reductionI(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
18860 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_INT); // src2
18861 match(Set dst (AddReductionVI src1 src2));
18862 match(Set dst (MulReductionVI src1 src2));
18863 match(Set dst (AndReductionV src1 src2));
18864 match(Set dst ( OrReductionV src1 src2));
18865 match(Set dst (XorReductionV src1 src2));
18866 match(Set dst (MinReductionV src1 src2));
18867 match(Set dst (MaxReductionV src1 src2));
18868 match(Set dst (UMinReductionV src1 src2));
18869 match(Set dst (UMaxReductionV src1 src2));
18870 effect(TEMP vtmp1, TEMP vtmp2);
18871 format %{ "vector_reduction_int $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
18872 ins_encode %{
18873 int opcode = this->ideal_Opcode();
18874 int vlen = Matcher::vector_length(this, $src2);
18875 __ reduceI(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
18876 %}
18877 ins_pipe( pipe_slow );
18878 %}
18879
18880 // =======================Long Reduction==========================================
18881
18882 instruct reductionL(rRegL dst, rRegL src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
18883 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG && !VM_Version::supports_avx512dq());
18884 match(Set dst (AddReductionVL src1 src2));
18885 match(Set dst (MulReductionVL src1 src2));
18886 match(Set dst (AndReductionV src1 src2));
18887 match(Set dst ( OrReductionV src1 src2));
18888 match(Set dst (XorReductionV src1 src2));
18889 match(Set dst (MinReductionV src1 src2));
18890 match(Set dst (MaxReductionV src1 src2));
18891 match(Set dst (UMinReductionV src1 src2));
18892 match(Set dst (UMaxReductionV src1 src2));
18893 effect(TEMP vtmp1, TEMP vtmp2);
18894 format %{ "vector_reduction_long $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
18895 ins_encode %{
18896 int opcode = this->ideal_Opcode();
18897 int vlen = Matcher::vector_length(this, $src2);
18898 __ reduceL(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
18899 %}
18900 ins_pipe( pipe_slow );
18901 %}
18902
18903 instruct reductionL_avx512dq(rRegL dst, rRegL src1, vec src2, vec vtmp1, vec vtmp2) %{
18904 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_LONG && VM_Version::supports_avx512dq());
18905 match(Set dst (AddReductionVL src1 src2));
18906 match(Set dst (MulReductionVL src1 src2));
18907 match(Set dst (AndReductionV src1 src2));
18908 match(Set dst ( OrReductionV src1 src2));
18909 match(Set dst (XorReductionV src1 src2));
18910 match(Set dst (MinReductionV src1 src2));
18911 match(Set dst (MaxReductionV src1 src2));
18912 match(Set dst (UMinReductionV src1 src2));
18913 match(Set dst (UMaxReductionV src1 src2));
18914 effect(TEMP vtmp1, TEMP vtmp2);
18915 format %{ "vector_reduction_long $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
18916 ins_encode %{
18917 int opcode = this->ideal_Opcode();
18918 int vlen = Matcher::vector_length(this, $src2);
18919 __ reduceL(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
18920 %}
18921 ins_pipe( pipe_slow );
18922 %}
18923
18924 // =======================Float Reduction==========================================
18925
18926 instruct reductionF128(regF dst, vec src, vec vtmp) %{
18927 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) <= 4); // src
18928 match(Set dst (AddReductionVF dst src));
18929 match(Set dst (MulReductionVF dst src));
18930 effect(TEMP dst, TEMP vtmp);
18931 format %{ "vector_reduction_float $dst,$src ; using $vtmp as TEMP" %}
18932 ins_encode %{
18933 int opcode = this->ideal_Opcode();
18934 int vlen = Matcher::vector_length(this, $src);
18935 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister);
18936 %}
18937 ins_pipe( pipe_slow );
18938 %}
18939
18940 instruct reduction8F(regF dst, vec src, vec vtmp1, vec vtmp2) %{
18941 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src
18942 match(Set dst (AddReductionVF dst src));
18943 match(Set dst (MulReductionVF dst src));
18944 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
18945 format %{ "vector_reduction_float $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
18946 ins_encode %{
18947 int opcode = this->ideal_Opcode();
18948 int vlen = Matcher::vector_length(this, $src);
18949 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
18950 %}
18951 ins_pipe( pipe_slow );
18952 %}
18953
18954 instruct reduction16F(regF dst, legVec src, legVec vtmp1, legVec vtmp2) %{
18955 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 16); // src
18956 match(Set dst (AddReductionVF dst src));
18957 match(Set dst (MulReductionVF dst src));
18958 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
18959 format %{ "vector_reduction_float $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
18960 ins_encode %{
18961 int opcode = this->ideal_Opcode();
18962 int vlen = Matcher::vector_length(this, $src);
18963 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
18964 %}
18965 ins_pipe( pipe_slow );
18966 %}
18967
18968
18969 instruct unordered_reduction2F(regF dst, regF src1, vec src2) %{
18970 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
18971 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
18972 // src1 contains reduction identity
18973 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 2); // src2
18974 match(Set dst (AddReductionVF src1 src2));
18975 match(Set dst (MulReductionVF src1 src2));
18976 effect(TEMP dst);
18977 format %{ "vector_reduction_float $dst,$src1,$src2 ;" %}
18978 ins_encode %{
18979 int opcode = this->ideal_Opcode();
18980 int vlen = Matcher::vector_length(this, $src2);
18981 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister);
18982 %}
18983 ins_pipe( pipe_slow );
18984 %}
18985
18986 instruct unordered_reduction4F(regF dst, regF src1, vec src2, vec vtmp) %{
18987 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
18988 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
18989 // src1 contains reduction identity
18990 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 4); // src2
18991 match(Set dst (AddReductionVF src1 src2));
18992 match(Set dst (MulReductionVF src1 src2));
18993 effect(TEMP dst, TEMP vtmp);
18994 format %{ "vector_reduction_float $dst,$src1,$src2 ; using $vtmp as TEMP" %}
18995 ins_encode %{
18996 int opcode = this->ideal_Opcode();
18997 int vlen = Matcher::vector_length(this, $src2);
18998 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp$$XMMRegister);
18999 %}
19000 ins_pipe( pipe_slow );
19001 %}
19002
19003 instruct unordered_reduction8F(regF dst, regF src1, vec src2, vec vtmp1, vec vtmp2) %{
19004 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
19005 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19006 // src1 contains reduction identity
19007 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src2
19008 match(Set dst (AddReductionVF src1 src2));
19009 match(Set dst (MulReductionVF src1 src2));
19010 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19011 format %{ "vector_reduction_float $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19012 ins_encode %{
19013 int opcode = this->ideal_Opcode();
19014 int vlen = Matcher::vector_length(this, $src2);
19015 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19016 %}
19017 ins_pipe( pipe_slow );
19018 %}
19019
19020 instruct unordered_reduction16F(regF dst, regF src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19021 // Non-strictly ordered floating-point add/mul reduction for floats. This rule is
19022 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19023 // src1 contains reduction identity
19024 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 16); // src2
19025 match(Set dst (AddReductionVF src1 src2));
19026 match(Set dst (MulReductionVF src1 src2));
19027 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19028 format %{ "vector_reduction_float $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19029 ins_encode %{
19030 int opcode = this->ideal_Opcode();
19031 int vlen = Matcher::vector_length(this, $src2);
19032 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19033 %}
19034 ins_pipe( pipe_slow );
19035 %}
19036
19037 // =======================Double Reduction==========================================
19038
19039 instruct reduction2D(regD dst, vec src, vec vtmp) %{
19040 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 2); // src
19041 match(Set dst (AddReductionVD dst src));
19042 match(Set dst (MulReductionVD dst src));
19043 effect(TEMP dst, TEMP vtmp);
19044 format %{ "vector_reduction_double $dst,$src ; using $vtmp as TEMP" %}
19045 ins_encode %{
19046 int opcode = this->ideal_Opcode();
19047 int vlen = Matcher::vector_length(this, $src);
19048 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp$$XMMRegister);
19049 %}
19050 ins_pipe( pipe_slow );
19051 %}
19052
19053 instruct reduction4D(regD dst, vec src, vec vtmp1, vec vtmp2) %{
19054 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 4); // src
19055 match(Set dst (AddReductionVD dst src));
19056 match(Set dst (MulReductionVD dst src));
19057 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19058 format %{ "vector_reduction_double $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
19059 ins_encode %{
19060 int opcode = this->ideal_Opcode();
19061 int vlen = Matcher::vector_length(this, $src);
19062 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19063 %}
19064 ins_pipe( pipe_slow );
19065 %}
19066
19067 instruct reduction8D(regD dst, legVec src, legVec vtmp1, legVec vtmp2) %{
19068 predicate(n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src
19069 match(Set dst (AddReductionVD dst src));
19070 match(Set dst (MulReductionVD dst src));
19071 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19072 format %{ "vector_reduction_double $dst,$src ; using $vtmp1, $vtmp2 as TEMP" %}
19073 ins_encode %{
19074 int opcode = this->ideal_Opcode();
19075 int vlen = Matcher::vector_length(this, $src);
19076 __ reduce_fp(opcode, vlen, $dst$$XMMRegister, $src$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19077 %}
19078 ins_pipe( pipe_slow );
19079 %}
19080
19081 instruct unordered_reduction2D(regD dst, regD src1, vec src2) %{
19082 // Non-strictly ordered floating-point add/mul reduction for doubles. This rule is
19083 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19084 // src1 contains reduction identity
19085 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 2); // src2
19086 match(Set dst (AddReductionVD src1 src2));
19087 match(Set dst (MulReductionVD src1 src2));
19088 effect(TEMP dst);
19089 format %{ "vector_reduction_double $dst,$src1,$src2 ;" %}
19090 ins_encode %{
19091 int opcode = this->ideal_Opcode();
19092 int vlen = Matcher::vector_length(this, $src2);
19093 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister);
19094 %}
19095 ins_pipe( pipe_slow );
19096 %}
19097
19098 instruct unordered_reduction4D(regD dst, regD src1, vec src2, vec vtmp) %{
19099 // Non-strictly ordered floating-point add/mul reduction for doubles. This rule is
19100 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19101 // src1 contains reduction identity
19102 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 4); // src2
19103 match(Set dst (AddReductionVD src1 src2));
19104 match(Set dst (MulReductionVD src1 src2));
19105 effect(TEMP dst, TEMP vtmp);
19106 format %{ "vector_reduction_double $dst,$src1,$src2 ; using $vtmp as TEMP" %}
19107 ins_encode %{
19108 int opcode = this->ideal_Opcode();
19109 int vlen = Matcher::vector_length(this, $src2);
19110 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp$$XMMRegister);
19111 %}
19112 ins_pipe( pipe_slow );
19113 %}
19114
19115 instruct unordered_reduction8D(regD dst, regD src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19116 // Non-strictly ordered floating-point add/mul reduction for doubles. This rule is
19117 // intended for the VectorAPI (which allows for non-strictly ordered add/mul reduction).
19118 // src1 contains reduction identity
19119 predicate(!n->as_Reduction()->requires_strict_order() && Matcher::vector_length(n->in(2)) == 8); // src2
19120 match(Set dst (AddReductionVD src1 src2));
19121 match(Set dst (MulReductionVD src1 src2));
19122 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19123 format %{ "vector_reduction_double $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19124 ins_encode %{
19125 int opcode = this->ideal_Opcode();
19126 int vlen = Matcher::vector_length(this, $src2);
19127 __ unordered_reduce_fp(opcode, vlen, $dst$$XMMRegister, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19128 %}
19129 ins_pipe( pipe_slow );
19130 %}
19131
19132 // =======================Byte Reduction==========================================
19133
19134 instruct reductionB(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19135 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE && !VM_Version::supports_avx512bw());
19136 match(Set dst (AddReductionVI src1 src2));
19137 match(Set dst (AndReductionV src1 src2));
19138 match(Set dst ( OrReductionV src1 src2));
19139 match(Set dst (XorReductionV src1 src2));
19140 match(Set dst (MinReductionV src1 src2));
19141 match(Set dst (MaxReductionV src1 src2));
19142 match(Set dst (UMinReductionV src1 src2));
19143 match(Set dst (UMaxReductionV src1 src2));
19144 effect(TEMP vtmp1, TEMP vtmp2);
19145 format %{ "vector_reduction_byte $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19146 ins_encode %{
19147 int opcode = this->ideal_Opcode();
19148 int vlen = Matcher::vector_length(this, $src2);
19149 __ reduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19150 %}
19151 ins_pipe( pipe_slow );
19152 %}
19153
19154 instruct reductionB_avx512bw(rRegI dst, rRegI src1, vec src2, vec vtmp1, vec vtmp2) %{
19155 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE && VM_Version::supports_avx512bw());
19156 match(Set dst (AddReductionVI src1 src2));
19157 match(Set dst (AndReductionV src1 src2));
19158 match(Set dst ( OrReductionV src1 src2));
19159 match(Set dst (XorReductionV src1 src2));
19160 match(Set dst (MinReductionV src1 src2));
19161 match(Set dst (MaxReductionV src1 src2));
19162 match(Set dst (UMinReductionV src1 src2));
19163 match(Set dst (UMaxReductionV src1 src2));
19164 effect(TEMP vtmp1, TEMP vtmp2);
19165 format %{ "vector_reduction_byte $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19166 ins_encode %{
19167 int opcode = this->ideal_Opcode();
19168 int vlen = Matcher::vector_length(this, $src2);
19169 __ reduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19170 %}
19171 ins_pipe( pipe_slow );
19172 %}
19173
19174 // =======================Short Reduction==========================================
19175
19176 instruct reductionS(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19177 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_SHORT); // src2
19178 match(Set dst (AddReductionVI src1 src2));
19179 match(Set dst (MulReductionVI src1 src2));
19180 match(Set dst (AndReductionV src1 src2));
19181 match(Set dst ( OrReductionV src1 src2));
19182 match(Set dst (XorReductionV src1 src2));
19183 match(Set dst (MinReductionV src1 src2));
19184 match(Set dst (MaxReductionV src1 src2));
19185 match(Set dst (UMinReductionV src1 src2));
19186 match(Set dst (UMaxReductionV src1 src2));
19187 effect(TEMP vtmp1, TEMP vtmp2);
19188 format %{ "vector_reduction_short $dst,$src1,$src2 ; using $vtmp1, $vtmp2 as TEMP" %}
19189 ins_encode %{
19190 int opcode = this->ideal_Opcode();
19191 int vlen = Matcher::vector_length(this, $src2);
19192 __ reduceS(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19193 %}
19194 ins_pipe( pipe_slow );
19195 %}
19196
19197 // =======================Mul Reduction==========================================
19198
19199 instruct mul_reductionB(rRegI dst, rRegI src1, vec src2, vec vtmp1, vec vtmp2) %{
19200 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE &&
19201 Matcher::vector_length(n->in(2)) <= 32); // src2
19202 match(Set dst (MulReductionVI src1 src2));
19203 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19204 format %{ "vector_mul_reduction_byte $dst,$src1,$src2; using $vtmp1, $vtmp2 as TEMP" %}
19205 ins_encode %{
19206 int opcode = this->ideal_Opcode();
19207 int vlen = Matcher::vector_length(this, $src2);
19208 __ mulreduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19209 %}
19210 ins_pipe( pipe_slow );
19211 %}
19212
19213 instruct mul_reduction64B(rRegI dst, rRegI src1, legVec src2, legVec vtmp1, legVec vtmp2) %{
19214 predicate(Matcher::vector_element_basic_type(n->in(2)) == T_BYTE &&
19215 Matcher::vector_length(n->in(2)) == 64); // src2
19216 match(Set dst (MulReductionVI src1 src2));
19217 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
19218 format %{ "vector_mul_reduction_byte $dst,$src1,$src2; using $vtmp1, $vtmp2 as TEMP" %}
19219 ins_encode %{
19220 int opcode = this->ideal_Opcode();
19221 int vlen = Matcher::vector_length(this, $src2);
19222 __ mulreduceB(opcode, vlen, $dst$$Register, $src1$$Register, $src2$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister);
19223 %}
19224 ins_pipe( pipe_slow );
19225 %}
19226
19227 //--------------------Min/Max Float Reduction --------------------
19228 // Float Min Reduction
19229 instruct minmax_reduction2F(legRegF dst, immF src1, legVec src2, legVec tmp, legVec atmp,
19230 legVec btmp, legVec xmm_1, rFlagsReg cr) %{
19231 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19232 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19233 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19234 Matcher::vector_length(n->in(2)) == 2);
19235 match(Set dst (MinReductionV src1 src2));
19236 match(Set dst (MaxReductionV src1 src2));
19237 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_1, KILL cr);
19238 format %{ "vector_minmax2F_reduction $dst,$src1,$src2 ; using $tmp, $atmp, $btmp, $xmm_1 as TEMP" %}
19239 ins_encode %{
19240 assert(UseAVX > 0, "sanity");
19241
19242 int opcode = this->ideal_Opcode();
19243 int vlen = Matcher::vector_length(this, $src2);
19244 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, $tmp$$XMMRegister,
19245 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_1$$XMMRegister);
19246 %}
19247 ins_pipe( pipe_slow );
19248 %}
19249
19250 instruct minmax_reductionF(legRegF dst, immF src1, legVec src2, legVec tmp, legVec atmp,
19251 legVec btmp, legVec xmm_0, legVec xmm_1, rFlagsReg cr) %{
19252 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19253 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19254 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19255 Matcher::vector_length(n->in(2)) >= 4);
19256 match(Set dst (MinReductionV src1 src2));
19257 match(Set dst (MaxReductionV src1 src2));
19258 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_0, TEMP xmm_1, KILL cr);
19259 format %{ "vector_minmaxF_reduction $dst,$src1,$src2 ; using $tmp, $atmp, $btmp, $xmm_0, $xmm_1 as TEMP" %}
19260 ins_encode %{
19261 assert(UseAVX > 0, "sanity");
19262
19263 int opcode = this->ideal_Opcode();
19264 int vlen = Matcher::vector_length(this, $src2);
19265 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, $tmp$$XMMRegister,
19266 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_0$$XMMRegister, $xmm_1$$XMMRegister);
19267 %}
19268 ins_pipe( pipe_slow );
19269 %}
19270
19271 instruct minmax_reduction2F_av(legRegF dst, legVec src, legVec tmp, legVec atmp,
19272 legVec btmp, legVec xmm_1, rFlagsReg cr) %{
19273 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19274 Matcher::vector_length(n->in(2)) == 2);
19275 match(Set dst (MinReductionV dst src));
19276 match(Set dst (MaxReductionV dst src));
19277 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_1, KILL cr);
19278 format %{ "vector_minmax2F_reduction $dst,$src ; using $tmp, $atmp, $btmp, $xmm_1 as TEMP" %}
19279 ins_encode %{
19280 assert(UseAVX > 0, "sanity");
19281
19282 int opcode = this->ideal_Opcode();
19283 int vlen = Matcher::vector_length(this, $src);
19284 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, $tmp$$XMMRegister,
19285 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_1$$XMMRegister);
19286 %}
19287 ins_pipe( pipe_slow );
19288 %}
19289
19290
19291 instruct minmax_reductionF_av(legRegF dst, legVec src, legVec tmp, legVec atmp, legVec btmp,
19292 legVec xmm_0, legVec xmm_1, rFlagsReg cr) %{
19293 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19294 Matcher::vector_length(n->in(2)) >= 4);
19295 match(Set dst (MinReductionV dst src));
19296 match(Set dst (MaxReductionV dst src));
19297 effect(TEMP dst, TEMP tmp, TEMP atmp, TEMP btmp, TEMP xmm_0, TEMP xmm_1, KILL cr);
19298 format %{ "vector_minmaxF_reduction $dst,$src ; using $tmp, $atmp, $btmp, $xmm_0, $xmm_1 as TEMP" %}
19299 ins_encode %{
19300 assert(UseAVX > 0, "sanity");
19301
19302 int opcode = this->ideal_Opcode();
19303 int vlen = Matcher::vector_length(this, $src);
19304 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, $tmp$$XMMRegister,
19305 $atmp$$XMMRegister, $btmp$$XMMRegister, $xmm_0$$XMMRegister, $xmm_1$$XMMRegister);
19306 %}
19307 ins_pipe( pipe_slow );
19308 %}
19309
19310 instruct minmax_reduction2F_avx10_2(regF dst, immF src1, vec src2, vec xtmp1) %{
19311 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19312 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19313 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19314 Matcher::vector_length(n->in(2)) == 2);
19315 match(Set dst (MinReductionV src1 src2));
19316 match(Set dst (MaxReductionV src1 src2));
19317 effect(TEMP dst, TEMP xtmp1);
19318 format %{ "vector_minmax_reduction $dst, $src1, $src2 \t; using $xtmp1 as TEMP" %}
19319 ins_encode %{
19320 int opcode = this->ideal_Opcode();
19321 int vlen = Matcher::vector_length(this, $src2);
19322 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister,
19323 xnoreg, xnoreg, xnoreg, $xtmp1$$XMMRegister);
19324 %}
19325 ins_pipe( pipe_slow );
19326 %}
19327
19328 instruct minmax_reductionF_avx10_2(regF dst, immF src1, vec src2, vec xtmp1, vec xtmp2) %{
19329 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19330 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeF::POS_INF) ||
19331 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeF::NEG_INF)) &&
19332 Matcher::vector_length(n->in(2)) >= 4);
19333 match(Set dst (MinReductionV src1 src2));
19334 match(Set dst (MaxReductionV src1 src2));
19335 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19336 format %{ "vector_minmax_reduction $dst, $src1, $src2 \t; using $xtmp1 and $xtmp2 as TEMP" %}
19337 ins_encode %{
19338 int opcode = this->ideal_Opcode();
19339 int vlen = Matcher::vector_length(this, $src2);
19340 __ reduceFloatMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, xnoreg, xnoreg,
19341 xnoreg, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19342 %}
19343 ins_pipe( pipe_slow );
19344 %}
19345
19346 instruct minmax_reduction2F_av_avx10_2(regF dst, vec src, vec xtmp1) %{
19347 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19348 Matcher::vector_length(n->in(2)) == 2);
19349 match(Set dst (MinReductionV dst src));
19350 match(Set dst (MaxReductionV dst src));
19351 effect(TEMP dst, TEMP xtmp1);
19352 format %{ "vector_minmax2F_reduction $dst, $src \t; using $xtmp1 as TEMP" %}
19353 ins_encode %{
19354 int opcode = this->ideal_Opcode();
19355 int vlen = Matcher::vector_length(this, $src);
19356 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, xnoreg, xnoreg, xnoreg,
19357 $xtmp1$$XMMRegister);
19358 %}
19359 ins_pipe( pipe_slow );
19360 %}
19361
19362 instruct minmax_reductionF_av_avx10_2(regF dst, vec src, vec xtmp1, vec xtmp2) %{
19363 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT &&
19364 Matcher::vector_length(n->in(2)) >= 4);
19365 match(Set dst (MinReductionV dst src));
19366 match(Set dst (MaxReductionV dst src));
19367 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19368 format %{ "vector_minmax2F_reduction $dst, $src \t; using $xtmp1 and $xtmp2 as TEMP" %}
19369 ins_encode %{
19370 int opcode = this->ideal_Opcode();
19371 int vlen = Matcher::vector_length(this, $src);
19372 __ reduceFloatMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister, xnoreg, xnoreg, xnoreg,
19373 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19374 %}
19375 ins_pipe( pipe_slow );
19376 %}
19377
19378 //--------------------Min Double Reduction --------------------
19379 instruct minmax_reduction2D(legRegD dst, immD src1, legVec src2, legVec tmp1, legVec tmp2,
19380 legVec tmp3, legVec tmp4, rFlagsReg cr) %{
19381 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19382 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19383 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19384 Matcher::vector_length(n->in(2)) == 2);
19385 match(Set dst (MinReductionV src1 src2));
19386 match(Set dst (MaxReductionV src1 src2));
19387 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
19388 format %{ "vector_minmax2D_reduction $dst,$src1,$src2 ; using $tmp1, $tmp2, $tmp3, $tmp4 as TEMP" %}
19389 ins_encode %{
19390 assert(UseAVX > 0, "sanity");
19391
19392 int opcode = this->ideal_Opcode();
19393 int vlen = Matcher::vector_length(this, $src2);
19394 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister,
19395 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister);
19396 %}
19397 ins_pipe( pipe_slow );
19398 %}
19399
19400 instruct minmax_reductionD(legRegD dst, immD src1, legVec src2, legVec tmp1, legVec tmp2,
19401 legVec tmp3, legVec tmp4, legVec tmp5, rFlagsReg cr) %{
19402 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19403 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19404 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19405 Matcher::vector_length(n->in(2)) >= 4);
19406 match(Set dst (MinReductionV src1 src2));
19407 match(Set dst (MaxReductionV src1 src2));
19408 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, KILL cr);
19409 format %{ "vector_minmaxD_reduction $dst,$src1,$src2 ; using $tmp1, $tmp2, $tmp3, $tmp4, $tmp5 as TEMP" %}
19410 ins_encode %{
19411 assert(UseAVX > 0, "sanity");
19412
19413 int opcode = this->ideal_Opcode();
19414 int vlen = Matcher::vector_length(this, $src2);
19415 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister,
19416 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister, $tmp5$$XMMRegister);
19417 %}
19418 ins_pipe( pipe_slow );
19419 %}
19420
19421
19422 instruct minmax_reduction2D_av(legRegD dst, legVec src, legVec tmp1, legVec tmp2,
19423 legVec tmp3, legVec tmp4, rFlagsReg cr) %{
19424 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19425 Matcher::vector_length(n->in(2)) == 2);
19426 match(Set dst (MinReductionV dst src));
19427 match(Set dst (MaxReductionV dst src));
19428 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
19429 format %{ "vector_minmax2D_reduction $dst,$src ; using $tmp1, $tmp2, $tmp3, $tmp4 as TEMP" %}
19430 ins_encode %{
19431 assert(UseAVX > 0, "sanity");
19432
19433 int opcode = this->ideal_Opcode();
19434 int vlen = Matcher::vector_length(this, $src);
19435 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19436 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister);
19437 %}
19438 ins_pipe( pipe_slow );
19439 %}
19440
19441 instruct minmax_reductionD_av(legRegD dst, legVec src, legVec tmp1, legVec tmp2, legVec tmp3,
19442 legVec tmp4, legVec tmp5, rFlagsReg cr) %{
19443 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19444 Matcher::vector_length(n->in(2)) >= 4);
19445 match(Set dst (MinReductionV dst src));
19446 match(Set dst (MaxReductionV dst src));
19447 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, KILL cr);
19448 format %{ "vector_minmaxD_reduction $dst,$src ; using $tmp1, $tmp2, $tmp3, $tmp4, $tmp5 as TEMP" %}
19449 ins_encode %{
19450 assert(UseAVX > 0, "sanity");
19451
19452 int opcode = this->ideal_Opcode();
19453 int vlen = Matcher::vector_length(this, $src);
19454 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19455 $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister, $tmp4$$XMMRegister, $tmp5$$XMMRegister);
19456 %}
19457 ins_pipe( pipe_slow );
19458 %}
19459
19460 instruct minmax_reduction2D_avx10_2(regD dst, immD src1, vec src2, vec xtmp1) %{
19461 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19462 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19463 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19464 Matcher::vector_length(n->in(2)) == 2);
19465 match(Set dst (MinReductionV src1 src2));
19466 match(Set dst (MaxReductionV src1 src2));
19467 effect(TEMP dst, TEMP xtmp1);
19468 format %{ "vector_minmax2D_reduction $dst, $src1, $src2 ; using $xtmp1 as TEMP" %}
19469 ins_encode %{
19470 int opcode = this->ideal_Opcode();
19471 int vlen = Matcher::vector_length(this, $src2);
19472 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, xnoreg,
19473 xnoreg, xnoreg, $xtmp1$$XMMRegister);
19474 %}
19475 ins_pipe( pipe_slow );
19476 %}
19477
19478 instruct minmax_reductionD_avx10_2(regD dst, immD src1, vec src2, vec xtmp1, vec xtmp2) %{
19479 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19480 ((n->Opcode() == Op_MinReductionV && n->in(1)->bottom_type() == TypeD::POS_INF) ||
19481 (n->Opcode() == Op_MaxReductionV && n->in(1)->bottom_type() == TypeD::NEG_INF)) &&
19482 Matcher::vector_length(n->in(2)) >= 4);
19483 match(Set dst (MinReductionV src1 src2));
19484 match(Set dst (MaxReductionV src1 src2));
19485 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19486 format %{ "vector_minmaxD_reduction $dst, $src1, $src2 ; using $xtmp1 and $xtmp2 as TEMP" %}
19487 ins_encode %{
19488 int opcode = this->ideal_Opcode();
19489 int vlen = Matcher::vector_length(this, $src2);
19490 __ reduceDoubleMinMax(opcode, vlen, false, $dst$$XMMRegister, $src2$$XMMRegister, xnoreg, xnoreg,
19491 xnoreg, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19492 %}
19493 ins_pipe( pipe_slow );
19494 %}
19495
19496
19497 instruct minmax_reduction2D_av_avx10_2(regD dst, vec src, vec xtmp1) %{
19498 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19499 Matcher::vector_length(n->in(2)) == 2);
19500 match(Set dst (MinReductionV dst src));
19501 match(Set dst (MaxReductionV dst src));
19502 effect(TEMP dst, TEMP xtmp1);
19503 format %{ "vector_minmax2D_reduction $dst, $src ; using $xtmp1 as TEMP" %}
19504 ins_encode %{
19505 int opcode = this->ideal_Opcode();
19506 int vlen = Matcher::vector_length(this, $src);
19507 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19508 xnoreg, xnoreg, xnoreg, $xtmp1$$XMMRegister);
19509 %}
19510 ins_pipe( pipe_slow );
19511 %}
19512
19513 instruct minmax_reductionD_av_avx10_2(regD dst, vec src, vec xtmp1, vec xtmp2) %{
19514 predicate(VM_Version::supports_avx10_2() && Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE &&
19515 Matcher::vector_length(n->in(2)) >= 4);
19516 match(Set dst (MinReductionV dst src));
19517 match(Set dst (MaxReductionV dst src));
19518 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
19519 format %{ "vector_minmaxD_reduction $dst, $src ; using $xtmp1 and $xtmp2 as TEMP" %}
19520 ins_encode %{
19521 int opcode = this->ideal_Opcode();
19522 int vlen = Matcher::vector_length(this, $src);
19523 __ reduceDoubleMinMax(opcode, vlen, true, $dst$$XMMRegister, $src$$XMMRegister,
19524 xnoreg, xnoreg, xnoreg, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
19525 %}
19526 ins_pipe( pipe_slow );
19527 %}
19528
19529 // ====================VECTOR ARITHMETIC=======================================
19530
19531 // --------------------------------- ADD --------------------------------------
19532
19533 // Bytes vector add
19534 instruct vaddB(vec dst, vec src) %{
19535 predicate(UseAVX == 0);
19536 match(Set dst (AddVB dst src));
19537 format %{ "paddb $dst,$src\t! add packedB" %}
19538 ins_encode %{
19539 __ paddb($dst$$XMMRegister, $src$$XMMRegister);
19540 %}
19541 ins_pipe( pipe_slow );
19542 %}
19543
19544 instruct vaddB_reg(vec dst, vec src1, vec src2) %{
19545 predicate(UseAVX > 0);
19546 match(Set dst (AddVB src1 src2));
19547 format %{ "vpaddb $dst,$src1,$src2\t! add packedB" %}
19548 ins_encode %{
19549 int vlen_enc = vector_length_encoding(this);
19550 __ vpaddb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19551 %}
19552 ins_pipe( pipe_slow );
19553 %}
19554
19555 instruct vaddB_mem(vec dst, vec src, memory mem) %{
19556 predicate((UseAVX > 0) &&
19557 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19558 match(Set dst (AddVB src (LoadVector mem)));
19559 format %{ "vpaddb $dst,$src,$mem\t! add packedB" %}
19560 ins_encode %{
19561 int vlen_enc = vector_length_encoding(this);
19562 __ vpaddb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19563 %}
19564 ins_pipe( pipe_slow );
19565 %}
19566
19567 // Shorts/Chars vector add
19568 instruct vaddS(vec dst, vec src) %{
19569 predicate(UseAVX == 0);
19570 match(Set dst (AddVS dst src));
19571 format %{ "paddw $dst,$src\t! add packedS" %}
19572 ins_encode %{
19573 __ paddw($dst$$XMMRegister, $src$$XMMRegister);
19574 %}
19575 ins_pipe( pipe_slow );
19576 %}
19577
19578 instruct vaddS_reg(vec dst, vec src1, vec src2) %{
19579 predicate(UseAVX > 0);
19580 match(Set dst (AddVS src1 src2));
19581 format %{ "vpaddw $dst,$src1,$src2\t! add packedS" %}
19582 ins_encode %{
19583 int vlen_enc = vector_length_encoding(this);
19584 __ vpaddw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19585 %}
19586 ins_pipe( pipe_slow );
19587 %}
19588
19589 instruct vaddS_mem(vec dst, vec src, memory mem) %{
19590 predicate((UseAVX > 0) &&
19591 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19592 match(Set dst (AddVS src (LoadVector mem)));
19593 format %{ "vpaddw $dst,$src,$mem\t! add packedS" %}
19594 ins_encode %{
19595 int vlen_enc = vector_length_encoding(this);
19596 __ vpaddw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19597 %}
19598 ins_pipe( pipe_slow );
19599 %}
19600
19601 // Integers vector add
19602 instruct vaddI(vec dst, vec src) %{
19603 predicate(UseAVX == 0);
19604 match(Set dst (AddVI dst src));
19605 format %{ "paddd $dst,$src\t! add packedI" %}
19606 ins_encode %{
19607 __ paddd($dst$$XMMRegister, $src$$XMMRegister);
19608 %}
19609 ins_pipe( pipe_slow );
19610 %}
19611
19612 instruct vaddI_reg(vec dst, vec src1, vec src2) %{
19613 predicate(UseAVX > 0);
19614 match(Set dst (AddVI src1 src2));
19615 format %{ "vpaddd $dst,$src1,$src2\t! add packedI" %}
19616 ins_encode %{
19617 int vlen_enc = vector_length_encoding(this);
19618 __ vpaddd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19619 %}
19620 ins_pipe( pipe_slow );
19621 %}
19622
19623
19624 instruct vaddI_mem(vec dst, vec src, memory mem) %{
19625 predicate((UseAVX > 0) &&
19626 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19627 match(Set dst (AddVI src (LoadVector mem)));
19628 format %{ "vpaddd $dst,$src,$mem\t! add packedI" %}
19629 ins_encode %{
19630 int vlen_enc = vector_length_encoding(this);
19631 __ vpaddd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19632 %}
19633 ins_pipe( pipe_slow );
19634 %}
19635
19636 // Longs vector add
19637 instruct vaddL(vec dst, vec src) %{
19638 predicate(UseAVX == 0);
19639 match(Set dst (AddVL dst src));
19640 format %{ "paddq $dst,$src\t! add packedL" %}
19641 ins_encode %{
19642 __ paddq($dst$$XMMRegister, $src$$XMMRegister);
19643 %}
19644 ins_pipe( pipe_slow );
19645 %}
19646
19647 instruct vaddL_reg(vec dst, vec src1, vec src2) %{
19648 predicate(UseAVX > 0);
19649 match(Set dst (AddVL src1 src2));
19650 format %{ "vpaddq $dst,$src1,$src2\t! add packedL" %}
19651 ins_encode %{
19652 int vlen_enc = vector_length_encoding(this);
19653 __ vpaddq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19654 %}
19655 ins_pipe( pipe_slow );
19656 %}
19657
19658 instruct vaddL_mem(vec dst, vec src, memory mem) %{
19659 predicate((UseAVX > 0) &&
19660 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19661 match(Set dst (AddVL src (LoadVector mem)));
19662 format %{ "vpaddq $dst,$src,$mem\t! add packedL" %}
19663 ins_encode %{
19664 int vlen_enc = vector_length_encoding(this);
19665 __ vpaddq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19666 %}
19667 ins_pipe( pipe_slow );
19668 %}
19669
19670 // Floats vector add
19671 instruct vaddF(vec dst, vec src) %{
19672 predicate(UseAVX == 0);
19673 match(Set dst (AddVF dst src));
19674 format %{ "addps $dst,$src\t! add packedF" %}
19675 ins_encode %{
19676 __ addps($dst$$XMMRegister, $src$$XMMRegister);
19677 %}
19678 ins_pipe( pipe_slow );
19679 %}
19680
19681 instruct vaddF_reg(vec dst, vec src1, vec src2) %{
19682 predicate(UseAVX > 0);
19683 match(Set dst (AddVF src1 src2));
19684 format %{ "vaddps $dst,$src1,$src2\t! add packedF" %}
19685 ins_encode %{
19686 int vlen_enc = vector_length_encoding(this);
19687 __ vaddps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19688 %}
19689 ins_pipe( pipe_slow );
19690 %}
19691
19692 instruct vaddF_mem(vec dst, vec src, memory mem) %{
19693 predicate((UseAVX > 0) &&
19694 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19695 match(Set dst (AddVF src (LoadVector mem)));
19696 format %{ "vaddps $dst,$src,$mem\t! add packedF" %}
19697 ins_encode %{
19698 int vlen_enc = vector_length_encoding(this);
19699 __ vaddps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19700 %}
19701 ins_pipe( pipe_slow );
19702 %}
19703
19704 // Doubles vector add
19705 instruct vaddD(vec dst, vec src) %{
19706 predicate(UseAVX == 0);
19707 match(Set dst (AddVD dst src));
19708 format %{ "addpd $dst,$src\t! add packedD" %}
19709 ins_encode %{
19710 __ addpd($dst$$XMMRegister, $src$$XMMRegister);
19711 %}
19712 ins_pipe( pipe_slow );
19713 %}
19714
19715 instruct vaddD_reg(vec dst, vec src1, vec src2) %{
19716 predicate(UseAVX > 0);
19717 match(Set dst (AddVD src1 src2));
19718 format %{ "vaddpd $dst,$src1,$src2\t! add packedD" %}
19719 ins_encode %{
19720 int vlen_enc = vector_length_encoding(this);
19721 __ vaddpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19722 %}
19723 ins_pipe( pipe_slow );
19724 %}
19725
19726 instruct vaddD_mem(vec dst, vec src, memory mem) %{
19727 predicate((UseAVX > 0) &&
19728 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19729 match(Set dst (AddVD src (LoadVector mem)));
19730 format %{ "vaddpd $dst,$src,$mem\t! add packedD" %}
19731 ins_encode %{
19732 int vlen_enc = vector_length_encoding(this);
19733 __ vaddpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19734 %}
19735 ins_pipe( pipe_slow );
19736 %}
19737
19738 // --------------------------------- SUB --------------------------------------
19739
19740 // Bytes vector sub
19741 instruct vsubB(vec dst, vec src) %{
19742 predicate(UseAVX == 0);
19743 match(Set dst (SubVB dst src));
19744 format %{ "psubb $dst,$src\t! sub packedB" %}
19745 ins_encode %{
19746 __ psubb($dst$$XMMRegister, $src$$XMMRegister);
19747 %}
19748 ins_pipe( pipe_slow );
19749 %}
19750
19751 instruct vsubB_reg(vec dst, vec src1, vec src2) %{
19752 predicate(UseAVX > 0);
19753 match(Set dst (SubVB src1 src2));
19754 format %{ "vpsubb $dst,$src1,$src2\t! sub packedB" %}
19755 ins_encode %{
19756 int vlen_enc = vector_length_encoding(this);
19757 __ vpsubb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19758 %}
19759 ins_pipe( pipe_slow );
19760 %}
19761
19762 instruct vsubB_mem(vec dst, vec src, memory mem) %{
19763 predicate((UseAVX > 0) &&
19764 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19765 match(Set dst (SubVB src (LoadVector mem)));
19766 format %{ "vpsubb $dst,$src,$mem\t! sub packedB" %}
19767 ins_encode %{
19768 int vlen_enc = vector_length_encoding(this);
19769 __ vpsubb($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19770 %}
19771 ins_pipe( pipe_slow );
19772 %}
19773
19774 // Shorts/Chars vector sub
19775 instruct vsubS(vec dst, vec src) %{
19776 predicate(UseAVX == 0);
19777 match(Set dst (SubVS dst src));
19778 format %{ "psubw $dst,$src\t! sub packedS" %}
19779 ins_encode %{
19780 __ psubw($dst$$XMMRegister, $src$$XMMRegister);
19781 %}
19782 ins_pipe( pipe_slow );
19783 %}
19784
19785
19786 instruct vsubS_reg(vec dst, vec src1, vec src2) %{
19787 predicate(UseAVX > 0);
19788 match(Set dst (SubVS src1 src2));
19789 format %{ "vpsubw $dst,$src1,$src2\t! sub packedS" %}
19790 ins_encode %{
19791 int vlen_enc = vector_length_encoding(this);
19792 __ vpsubw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19793 %}
19794 ins_pipe( pipe_slow );
19795 %}
19796
19797 instruct vsubS_mem(vec dst, vec src, memory mem) %{
19798 predicate((UseAVX > 0) &&
19799 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19800 match(Set dst (SubVS src (LoadVector mem)));
19801 format %{ "vpsubw $dst,$src,$mem\t! sub packedS" %}
19802 ins_encode %{
19803 int vlen_enc = vector_length_encoding(this);
19804 __ vpsubw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19805 %}
19806 ins_pipe( pipe_slow );
19807 %}
19808
19809 // Integers vector sub
19810 instruct vsubI(vec dst, vec src) %{
19811 predicate(UseAVX == 0);
19812 match(Set dst (SubVI dst src));
19813 format %{ "psubd $dst,$src\t! sub packedI" %}
19814 ins_encode %{
19815 __ psubd($dst$$XMMRegister, $src$$XMMRegister);
19816 %}
19817 ins_pipe( pipe_slow );
19818 %}
19819
19820 instruct vsubI_reg(vec dst, vec src1, vec src2) %{
19821 predicate(UseAVX > 0);
19822 match(Set dst (SubVI src1 src2));
19823 format %{ "vpsubd $dst,$src1,$src2\t! sub packedI" %}
19824 ins_encode %{
19825 int vlen_enc = vector_length_encoding(this);
19826 __ vpsubd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19827 %}
19828 ins_pipe( pipe_slow );
19829 %}
19830
19831 instruct vsubI_mem(vec dst, vec src, memory mem) %{
19832 predicate((UseAVX > 0) &&
19833 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19834 match(Set dst (SubVI src (LoadVector mem)));
19835 format %{ "vpsubd $dst,$src,$mem\t! sub packedI" %}
19836 ins_encode %{
19837 int vlen_enc = vector_length_encoding(this);
19838 __ vpsubd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19839 %}
19840 ins_pipe( pipe_slow );
19841 %}
19842
19843 // Longs vector sub
19844 instruct vsubL(vec dst, vec src) %{
19845 predicate(UseAVX == 0);
19846 match(Set dst (SubVL dst src));
19847 format %{ "psubq $dst,$src\t! sub packedL" %}
19848 ins_encode %{
19849 __ psubq($dst$$XMMRegister, $src$$XMMRegister);
19850 %}
19851 ins_pipe( pipe_slow );
19852 %}
19853
19854 instruct vsubL_reg(vec dst, vec src1, vec src2) %{
19855 predicate(UseAVX > 0);
19856 match(Set dst (SubVL src1 src2));
19857 format %{ "vpsubq $dst,$src1,$src2\t! sub packedL" %}
19858 ins_encode %{
19859 int vlen_enc = vector_length_encoding(this);
19860 __ vpsubq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19861 %}
19862 ins_pipe( pipe_slow );
19863 %}
19864
19865
19866 instruct vsubL_mem(vec dst, vec src, memory mem) %{
19867 predicate((UseAVX > 0) &&
19868 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19869 match(Set dst (SubVL src (LoadVector mem)));
19870 format %{ "vpsubq $dst,$src,$mem\t! sub packedL" %}
19871 ins_encode %{
19872 int vlen_enc = vector_length_encoding(this);
19873 __ vpsubq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19874 %}
19875 ins_pipe( pipe_slow );
19876 %}
19877
19878 // Floats vector sub
19879 instruct vsubF(vec dst, vec src) %{
19880 predicate(UseAVX == 0);
19881 match(Set dst (SubVF dst src));
19882 format %{ "subps $dst,$src\t! sub packedF" %}
19883 ins_encode %{
19884 __ subps($dst$$XMMRegister, $src$$XMMRegister);
19885 %}
19886 ins_pipe( pipe_slow );
19887 %}
19888
19889 instruct vsubF_reg(vec dst, vec src1, vec src2) %{
19890 predicate(UseAVX > 0);
19891 match(Set dst (SubVF src1 src2));
19892 format %{ "vsubps $dst,$src1,$src2\t! sub packedF" %}
19893 ins_encode %{
19894 int vlen_enc = vector_length_encoding(this);
19895 __ vsubps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19896 %}
19897 ins_pipe( pipe_slow );
19898 %}
19899
19900 instruct vsubF_mem(vec dst, vec src, memory mem) %{
19901 predicate((UseAVX > 0) &&
19902 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19903 match(Set dst (SubVF src (LoadVector mem)));
19904 format %{ "vsubps $dst,$src,$mem\t! sub packedF" %}
19905 ins_encode %{
19906 int vlen_enc = vector_length_encoding(this);
19907 __ vsubps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19908 %}
19909 ins_pipe( pipe_slow );
19910 %}
19911
19912 // Doubles vector sub
19913 instruct vsubD(vec dst, vec src) %{
19914 predicate(UseAVX == 0);
19915 match(Set dst (SubVD dst src));
19916 format %{ "subpd $dst,$src\t! sub packedD" %}
19917 ins_encode %{
19918 __ subpd($dst$$XMMRegister, $src$$XMMRegister);
19919 %}
19920 ins_pipe( pipe_slow );
19921 %}
19922
19923 instruct vsubD_reg(vec dst, vec src1, vec src2) %{
19924 predicate(UseAVX > 0);
19925 match(Set dst (SubVD src1 src2));
19926 format %{ "vsubpd $dst,$src1,$src2\t! sub packedD" %}
19927 ins_encode %{
19928 int vlen_enc = vector_length_encoding(this);
19929 __ vsubpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
19930 %}
19931 ins_pipe( pipe_slow );
19932 %}
19933
19934 instruct vsubD_mem(vec dst, vec src, memory mem) %{
19935 predicate((UseAVX > 0) &&
19936 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
19937 match(Set dst (SubVD src (LoadVector mem)));
19938 format %{ "vsubpd $dst,$src,$mem\t! sub packedD" %}
19939 ins_encode %{
19940 int vlen_enc = vector_length_encoding(this);
19941 __ vsubpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
19942 %}
19943 ins_pipe( pipe_slow );
19944 %}
19945
19946 // --------------------------------- MUL --------------------------------------
19947
19948 // Byte vector mul
19949 instruct vmul8B(vec dst, vec src1, vec src2, vec xtmp) %{
19950 predicate(Matcher::vector_length_in_bytes(n) <= 8);
19951 match(Set dst (MulVB src1 src2));
19952 effect(TEMP dst, TEMP xtmp);
19953 format %{ "mulVB $dst, $src1, $src2\t! using $xtmp as TEMP" %}
19954 ins_encode %{
19955 assert(UseSSE > 3, "required");
19956 __ pmovsxbw($dst$$XMMRegister, $src1$$XMMRegister);
19957 __ pmovsxbw($xtmp$$XMMRegister, $src2$$XMMRegister);
19958 __ pmullw($dst$$XMMRegister, $xtmp$$XMMRegister);
19959 __ psllw($dst$$XMMRegister, 8);
19960 __ psrlw($dst$$XMMRegister, 8);
19961 __ packuswb($dst$$XMMRegister, $dst$$XMMRegister);
19962 %}
19963 ins_pipe( pipe_slow );
19964 %}
19965
19966 instruct vmulB(vec dst, vec src1, vec src2, vec xtmp) %{
19967 predicate(UseAVX == 0 && Matcher::vector_length_in_bytes(n) > 8);
19968 match(Set dst (MulVB src1 src2));
19969 effect(TEMP dst, TEMP xtmp);
19970 format %{ "mulVB $dst, $src1, $src2\t! using $xtmp as TEMP" %}
19971 ins_encode %{
19972 assert(UseSSE > 3, "required");
19973 // Odd-index elements
19974 __ movdqu($dst$$XMMRegister, $src1$$XMMRegister);
19975 __ psrlw($dst$$XMMRegister, 8);
19976 __ movdqu($xtmp$$XMMRegister, $src2$$XMMRegister);
19977 __ psrlw($xtmp$$XMMRegister, 8);
19978 __ pmullw($dst$$XMMRegister, $xtmp$$XMMRegister);
19979 __ psllw($dst$$XMMRegister, 8);
19980 // Even-index elements
19981 __ movdqu($xtmp$$XMMRegister, $src1$$XMMRegister);
19982 __ pmullw($xtmp$$XMMRegister, $src2$$XMMRegister);
19983 __ psllw($xtmp$$XMMRegister, 8);
19984 __ psrlw($xtmp$$XMMRegister, 8);
19985 // Combine
19986 __ por($dst$$XMMRegister, $xtmp$$XMMRegister);
19987 %}
19988 ins_pipe( pipe_slow );
19989 %}
19990
19991 instruct vmulB_reg(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2) %{
19992 predicate(UseAVX > 0 && Matcher::vector_length_in_bytes(n) > 8);
19993 match(Set dst (MulVB src1 src2));
19994 effect(TEMP xtmp1, TEMP xtmp2);
19995 format %{ "vmulVB $dst, $src1, $src2\t! using $xtmp1, $xtmp2 as TEMP" %}
19996 ins_encode %{
19997 int vlen_enc = vector_length_encoding(this);
19998 // Odd-index elements
19999 __ vpsrlw($xtmp2$$XMMRegister, $src1$$XMMRegister, 8, vlen_enc);
20000 __ vpsrlw($xtmp1$$XMMRegister, $src2$$XMMRegister, 8, vlen_enc);
20001 __ vpmullw($xtmp2$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20002 __ vpsllw($xtmp2$$XMMRegister, $xtmp2$$XMMRegister, 8, vlen_enc);
20003 // Even-index elements
20004 __ vpmullw($xtmp1$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20005 __ vpsllw($xtmp1$$XMMRegister, $xtmp1$$XMMRegister, 8, vlen_enc);
20006 __ vpsrlw($xtmp1$$XMMRegister, $xtmp1$$XMMRegister, 8, vlen_enc);
20007 // Combine
20008 __ vpor($dst$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20009 %}
20010 ins_pipe( pipe_slow );
20011 %}
20012
20013 // Shorts/Chars vector mul
20014 instruct vmulS(vec dst, vec src) %{
20015 predicate(UseAVX == 0);
20016 match(Set dst (MulVS dst src));
20017 format %{ "pmullw $dst,$src\t! mul packedS" %}
20018 ins_encode %{
20019 __ pmullw($dst$$XMMRegister, $src$$XMMRegister);
20020 %}
20021 ins_pipe( pipe_slow );
20022 %}
20023
20024 instruct vmulS_reg(vec dst, vec src1, vec src2) %{
20025 predicate(UseAVX > 0);
20026 match(Set dst (MulVS src1 src2));
20027 format %{ "vpmullw $dst,$src1,$src2\t! mul packedS" %}
20028 ins_encode %{
20029 int vlen_enc = vector_length_encoding(this);
20030 __ vpmullw($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20031 %}
20032 ins_pipe( pipe_slow );
20033 %}
20034
20035 instruct vmulS_mem(vec dst, vec src, memory mem) %{
20036 predicate((UseAVX > 0) &&
20037 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20038 match(Set dst (MulVS src (LoadVector mem)));
20039 format %{ "vpmullw $dst,$src,$mem\t! mul packedS" %}
20040 ins_encode %{
20041 int vlen_enc = vector_length_encoding(this);
20042 __ vpmullw($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20043 %}
20044 ins_pipe( pipe_slow );
20045 %}
20046
20047 // Integers vector mul
20048 instruct vmulI(vec dst, vec src) %{
20049 predicate(UseAVX == 0);
20050 match(Set dst (MulVI dst src));
20051 format %{ "pmulld $dst,$src\t! mul packedI" %}
20052 ins_encode %{
20053 assert(UseSSE > 3, "required");
20054 __ pmulld($dst$$XMMRegister, $src$$XMMRegister);
20055 %}
20056 ins_pipe( pipe_slow );
20057 %}
20058
20059 instruct vmulI_reg(vec dst, vec src1, vec src2) %{
20060 predicate(UseAVX > 0);
20061 match(Set dst (MulVI src1 src2));
20062 format %{ "vpmulld $dst,$src1,$src2\t! mul packedI" %}
20063 ins_encode %{
20064 int vlen_enc = vector_length_encoding(this);
20065 __ vpmulld($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20066 %}
20067 ins_pipe( pipe_slow );
20068 %}
20069
20070 instruct vmulI_mem(vec dst, vec src, memory mem) %{
20071 predicate((UseAVX > 0) &&
20072 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20073 match(Set dst (MulVI src (LoadVector mem)));
20074 format %{ "vpmulld $dst,$src,$mem\t! mul packedI" %}
20075 ins_encode %{
20076 int vlen_enc = vector_length_encoding(this);
20077 __ vpmulld($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20078 %}
20079 ins_pipe( pipe_slow );
20080 %}
20081
20082 // Longs vector mul
20083 instruct evmulL_reg(vec dst, vec src1, vec src2) %{
20084 predicate((Matcher::vector_length_in_bytes(n) == 64 &&
20085 VM_Version::supports_avx512dq()) ||
20086 VM_Version::supports_avx512vldq());
20087 match(Set dst (MulVL src1 src2));
20088 ins_cost(500);
20089 format %{ "evpmullq $dst,$src1,$src2\t! mul packedL" %}
20090 ins_encode %{
20091 assert(UseAVX > 2, "required");
20092 int vlen_enc = vector_length_encoding(this);
20093 __ evpmullq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20094 %}
20095 ins_pipe( pipe_slow );
20096 %}
20097
20098 instruct evmulL_mem(vec dst, vec src, memory mem) %{
20099 predicate((Matcher::vector_length_in_bytes(n) == 64 &&
20100 VM_Version::supports_avx512dq()) ||
20101 (Matcher::vector_length_in_bytes(n) > 8 &&
20102 VM_Version::supports_avx512vldq()));
20103 match(Set dst (MulVL src (LoadVector mem)));
20104 format %{ "evpmullq $dst,$src,$mem\t! mul packedL" %}
20105 ins_cost(500);
20106 ins_encode %{
20107 assert(UseAVX > 2, "required");
20108 int vlen_enc = vector_length_encoding(this);
20109 __ evpmullq($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20110 %}
20111 ins_pipe( pipe_slow );
20112 %}
20113
20114 instruct vmulL(vec dst, vec src1, vec src2, vec xtmp) %{
20115 predicate(UseAVX == 0);
20116 match(Set dst (MulVL src1 src2));
20117 ins_cost(500);
20118 effect(TEMP dst, TEMP xtmp);
20119 format %{ "mulVL $dst, $src1, $src2\t! using $xtmp as TEMP" %}
20120 ins_encode %{
20121 assert(VM_Version::supports_sse4_1(), "required");
20122 // Get the lo-hi products, only the lower 32 bits is in concerns
20123 __ pshufd($xtmp$$XMMRegister, $src2$$XMMRegister, 0xB1);
20124 __ pmulld($xtmp$$XMMRegister, $src1$$XMMRegister);
20125 __ pshufd($dst$$XMMRegister, $xtmp$$XMMRegister, 0xB1);
20126 __ paddd($dst$$XMMRegister, $xtmp$$XMMRegister);
20127 __ psllq($dst$$XMMRegister, 32);
20128 // Get the lo-lo products
20129 __ movdqu($xtmp$$XMMRegister, $src1$$XMMRegister);
20130 __ pmuludq($xtmp$$XMMRegister, $src2$$XMMRegister);
20131 __ paddq($dst$$XMMRegister, $xtmp$$XMMRegister);
20132 %}
20133 ins_pipe( pipe_slow );
20134 %}
20135
20136 instruct vmulL_reg(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2) %{
20137 predicate(UseAVX > 0 &&
20138 ((Matcher::vector_length_in_bytes(n) == 64 &&
20139 !VM_Version::supports_avx512dq()) ||
20140 (Matcher::vector_length_in_bytes(n) < 64 &&
20141 !VM_Version::supports_avx512vldq())));
20142 match(Set dst (MulVL src1 src2));
20143 effect(TEMP xtmp1, TEMP xtmp2);
20144 ins_cost(500);
20145 format %{ "vmulVL $dst, $src1, $src2\t! using $xtmp1, $xtmp2 as TEMP" %}
20146 ins_encode %{
20147 int vlen_enc = vector_length_encoding(this);
20148 // Get the lo-hi products, only the lower 32 bits is in concerns
20149 __ vpshufd($xtmp1$$XMMRegister, $src2$$XMMRegister, 0xB1, vlen_enc);
20150 __ vpmulld($xtmp1$$XMMRegister, $src1$$XMMRegister, $xtmp1$$XMMRegister, vlen_enc);
20151 __ vpshufd($xtmp2$$XMMRegister, $xtmp1$$XMMRegister, 0xB1, vlen_enc);
20152 __ vpaddd($xtmp2$$XMMRegister, $xtmp2$$XMMRegister, $xtmp1$$XMMRegister, vlen_enc);
20153 __ vpsllq($xtmp2$$XMMRegister, $xtmp2$$XMMRegister, 32, vlen_enc);
20154 // Get the lo-lo products
20155 __ vpmuludq($xtmp1$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20156 __ vpaddq($dst$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20157 %}
20158 ins_pipe( pipe_slow );
20159 %}
20160
20161 instruct vmuludq_reg(vec dst, vec src1, vec src2) %{
20162 predicate(UseAVX > 0 && n->as_MulVL()->has_uint_inputs());
20163 match(Set dst (MulVL src1 src2));
20164 ins_cost(100);
20165 format %{ "vpmuludq $dst,$src1,$src2\t! muludq packedL" %}
20166 ins_encode %{
20167 int vlen_enc = vector_length_encoding(this);
20168 __ vpmuludq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20169 %}
20170 ins_pipe( pipe_slow );
20171 %}
20172
20173 instruct vmuldq_reg(vec dst, vec src1, vec src2) %{
20174 predicate(UseAVX > 0 && n->as_MulVL()->has_int_inputs());
20175 match(Set dst (MulVL src1 src2));
20176 ins_cost(100);
20177 format %{ "vpmuldq $dst,$src1,$src2\t! muldq packedL" %}
20178 ins_encode %{
20179 int vlen_enc = vector_length_encoding(this);
20180 __ vpmuldq($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20181 %}
20182 ins_pipe( pipe_slow );
20183 %}
20184
20185 // Floats vector mul
20186 instruct vmulF(vec dst, vec src) %{
20187 predicate(UseAVX == 0);
20188 match(Set dst (MulVF dst src));
20189 format %{ "mulps $dst,$src\t! mul packedF" %}
20190 ins_encode %{
20191 __ mulps($dst$$XMMRegister, $src$$XMMRegister);
20192 %}
20193 ins_pipe( pipe_slow );
20194 %}
20195
20196 instruct vmulF_reg(vec dst, vec src1, vec src2) %{
20197 predicate(UseAVX > 0);
20198 match(Set dst (MulVF src1 src2));
20199 format %{ "vmulps $dst,$src1,$src2\t! mul packedF" %}
20200 ins_encode %{
20201 int vlen_enc = vector_length_encoding(this);
20202 __ vmulps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20203 %}
20204 ins_pipe( pipe_slow );
20205 %}
20206
20207 instruct vmulF_mem(vec dst, vec src, memory mem) %{
20208 predicate((UseAVX > 0) &&
20209 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20210 match(Set dst (MulVF src (LoadVector mem)));
20211 format %{ "vmulps $dst,$src,$mem\t! mul packedF" %}
20212 ins_encode %{
20213 int vlen_enc = vector_length_encoding(this);
20214 __ vmulps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20215 %}
20216 ins_pipe( pipe_slow );
20217 %}
20218
20219 // Doubles vector mul
20220 instruct vmulD(vec dst, vec src) %{
20221 predicate(UseAVX == 0);
20222 match(Set dst (MulVD dst src));
20223 format %{ "mulpd $dst,$src\t! mul packedD" %}
20224 ins_encode %{
20225 __ mulpd($dst$$XMMRegister, $src$$XMMRegister);
20226 %}
20227 ins_pipe( pipe_slow );
20228 %}
20229
20230 instruct vmulD_reg(vec dst, vec src1, vec src2) %{
20231 predicate(UseAVX > 0);
20232 match(Set dst (MulVD src1 src2));
20233 format %{ "vmulpd $dst,$src1,$src2\t! mul packedD" %}
20234 ins_encode %{
20235 int vlen_enc = vector_length_encoding(this);
20236 __ vmulpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20237 %}
20238 ins_pipe( pipe_slow );
20239 %}
20240
20241 instruct vmulD_mem(vec dst, vec src, memory mem) %{
20242 predicate((UseAVX > 0) &&
20243 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20244 match(Set dst (MulVD src (LoadVector mem)));
20245 format %{ "vmulpd $dst,$src,$mem\t! mul packedD" %}
20246 ins_encode %{
20247 int vlen_enc = vector_length_encoding(this);
20248 __ vmulpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20249 %}
20250 ins_pipe( pipe_slow );
20251 %}
20252
20253 // --------------------------------- DIV --------------------------------------
20254
20255 // Floats vector div
20256 instruct vdivF(vec dst, vec src) %{
20257 predicate(UseAVX == 0);
20258 match(Set dst (DivVF dst src));
20259 format %{ "divps $dst,$src\t! div packedF" %}
20260 ins_encode %{
20261 __ divps($dst$$XMMRegister, $src$$XMMRegister);
20262 %}
20263 ins_pipe( pipe_slow );
20264 %}
20265
20266 instruct vdivF_reg(vec dst, vec src1, vec src2) %{
20267 predicate(UseAVX > 0);
20268 match(Set dst (DivVF src1 src2));
20269 format %{ "vdivps $dst,$src1,$src2\t! div packedF" %}
20270 ins_encode %{
20271 int vlen_enc = vector_length_encoding(this);
20272 __ vdivps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20273 %}
20274 ins_pipe( pipe_slow );
20275 %}
20276
20277 instruct vdivF_mem(vec dst, vec src, memory mem) %{
20278 predicate((UseAVX > 0) &&
20279 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20280 match(Set dst (DivVF src (LoadVector mem)));
20281 format %{ "vdivps $dst,$src,$mem\t! div packedF" %}
20282 ins_encode %{
20283 int vlen_enc = vector_length_encoding(this);
20284 __ vdivps($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20285 %}
20286 ins_pipe( pipe_slow );
20287 %}
20288
20289 // Doubles vector div
20290 instruct vdivD(vec dst, vec src) %{
20291 predicate(UseAVX == 0);
20292 match(Set dst (DivVD dst src));
20293 format %{ "divpd $dst,$src\t! div packedD" %}
20294 ins_encode %{
20295 __ divpd($dst$$XMMRegister, $src$$XMMRegister);
20296 %}
20297 ins_pipe( pipe_slow );
20298 %}
20299
20300 instruct vdivD_reg(vec dst, vec src1, vec src2) %{
20301 predicate(UseAVX > 0);
20302 match(Set dst (DivVD src1 src2));
20303 format %{ "vdivpd $dst,$src1,$src2\t! div packedD" %}
20304 ins_encode %{
20305 int vlen_enc = vector_length_encoding(this);
20306 __ vdivpd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20307 %}
20308 ins_pipe( pipe_slow );
20309 %}
20310
20311 instruct vdivD_mem(vec dst, vec src, memory mem) %{
20312 predicate((UseAVX > 0) &&
20313 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
20314 match(Set dst (DivVD src (LoadVector mem)));
20315 format %{ "vdivpd $dst,$src,$mem\t! div packedD" %}
20316 ins_encode %{
20317 int vlen_enc = vector_length_encoding(this);
20318 __ vdivpd($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
20319 %}
20320 ins_pipe( pipe_slow );
20321 %}
20322
20323 // ------------------------------ MinMax ---------------------------------------
20324
20325 // Byte, Short, Int vector Min/Max
20326 instruct minmax_reg_sse(vec dst, vec src) %{
20327 predicate(is_integral_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_element_basic_type(n) != T_LONG && // T_BYTE, T_SHORT, T_INT
20328 UseAVX == 0);
20329 match(Set dst (MinV dst src));
20330 match(Set dst (MaxV dst src));
20331 format %{ "vector_minmax $dst,$src\t! " %}
20332 ins_encode %{
20333 assert(UseSSE >= 4, "required");
20334
20335 int opcode = this->ideal_Opcode();
20336 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20337 __ pminmax(opcode, elem_bt, $dst$$XMMRegister, $src$$XMMRegister);
20338 %}
20339 ins_pipe( pipe_slow );
20340 %}
20341
20342 instruct vminmax_reg(vec dst, vec src1, vec src2) %{
20343 predicate(is_integral_type(Matcher::vector_element_basic_type(n)) && Matcher::vector_element_basic_type(n) != T_LONG && // T_BYTE, T_SHORT, T_INT
20344 UseAVX > 0);
20345 match(Set dst (MinV src1 src2));
20346 match(Set dst (MaxV src1 src2));
20347 format %{ "vector_minmax $dst,$src1,$src2\t! " %}
20348 ins_encode %{
20349 int opcode = this->ideal_Opcode();
20350 int vlen_enc = vector_length_encoding(this);
20351 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20352
20353 __ vpminmax(opcode, elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20354 %}
20355 ins_pipe( pipe_slow );
20356 %}
20357
20358 // Long vector Min/Max
20359 instruct minmaxL_reg_sse(vec dst, vec src, rxmm0 tmp) %{
20360 predicate(Matcher::vector_length_in_bytes(n) == 16 && Matcher::vector_element_basic_type(n) == T_LONG &&
20361 UseAVX == 0);
20362 match(Set dst (MinV dst src));
20363 match(Set dst (MaxV src dst));
20364 effect(TEMP dst, TEMP tmp);
20365 format %{ "vector_minmaxL $dst,$src\t!using $tmp as TEMP" %}
20366 ins_encode %{
20367 assert(UseSSE >= 4, "required");
20368
20369 int opcode = this->ideal_Opcode();
20370 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20371 assert(elem_bt == T_LONG, "sanity");
20372
20373 __ pminmax(opcode, elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $tmp$$XMMRegister);
20374 %}
20375 ins_pipe( pipe_slow );
20376 %}
20377
20378 instruct vminmaxL_reg_avx(legVec dst, legVec src1, legVec src2) %{
20379 predicate(Matcher::vector_length_in_bytes(n) <= 32 && Matcher::vector_element_basic_type(n) == T_LONG &&
20380 UseAVX > 0 && !VM_Version::supports_avx512vl());
20381 match(Set dst (MinV src1 src2));
20382 match(Set dst (MaxV src1 src2));
20383 effect(TEMP dst);
20384 format %{ "vector_minmaxL $dst,$src1,$src2\t! " %}
20385 ins_encode %{
20386 int vlen_enc = vector_length_encoding(this);
20387 int opcode = this->ideal_Opcode();
20388 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20389 assert(elem_bt == T_LONG, "sanity");
20390
20391 __ vpminmax(opcode, elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20392 %}
20393 ins_pipe( pipe_slow );
20394 %}
20395
20396 instruct vminmaxL_reg_evex(vec dst, vec src1, vec src2) %{
20397 predicate((Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()) &&
20398 Matcher::vector_element_basic_type(n) == T_LONG);
20399 match(Set dst (MinV src1 src2));
20400 match(Set dst (MaxV src1 src2));
20401 format %{ "vector_minmaxL $dst,$src1,src2\t! " %}
20402 ins_encode %{
20403 assert(UseAVX > 2, "required");
20404
20405 int vlen_enc = vector_length_encoding(this);
20406 int opcode = this->ideal_Opcode();
20407 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20408 assert(elem_bt == T_LONG, "sanity");
20409
20410 __ vpminmax(opcode, elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
20411 %}
20412 ins_pipe( pipe_slow );
20413 %}
20414
20415 // Float/Double vector Min/Max
20416 instruct minmaxFP_reg_avx10_2(vec dst, vec a, vec b) %{
20417 predicate(VM_Version::supports_avx10_2() &&
20418 is_floating_point_type(Matcher::vector_element_basic_type(n))); // T_FLOAT, T_DOUBLE
20419 match(Set dst (MinV a b));
20420 match(Set dst (MaxV a b));
20421 format %{ "vector_minmaxFP $dst, $a, $b" %}
20422 ins_encode %{
20423 int vlen_enc = vector_length_encoding(this);
20424 int opcode = this->ideal_Opcode();
20425 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20426 __ vminmax_fp_avx10_2(opcode, elem_bt, $dst$$XMMRegister, k0, $a$$XMMRegister, $b$$XMMRegister, vlen_enc);
20427 %}
20428 ins_pipe( pipe_slow );
20429 %}
20430
20431 // Float/Double vector Min/Max
20432 instruct minmaxFP_reg(legVec dst, legVec a, legVec b, legVec tmp, legVec atmp, legVec btmp) %{
20433 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_length_in_bytes(n) <= 32 &&
20434 is_floating_point_type(Matcher::vector_element_basic_type(n)) && // T_FLOAT, T_DOUBLE
20435 UseAVX > 0);
20436 match(Set dst (MinV a b));
20437 match(Set dst (MaxV a b));
20438 effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
20439 format %{ "vector_minmaxFP $dst,$a,$b\t!using $tmp, $atmp, $btmp as TEMP" %}
20440 ins_encode %{
20441 assert(UseAVX > 0, "required");
20442
20443 int opcode = this->ideal_Opcode();
20444 int vlen_enc = vector_length_encoding(this);
20445 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20446
20447 __ vminmax_fp(opcode, elem_bt,
20448 $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister,
20449 $tmp$$XMMRegister, $atmp$$XMMRegister , $btmp$$XMMRegister, vlen_enc);
20450 %}
20451 ins_pipe( pipe_slow );
20452 %}
20453
20454 instruct evminmaxFP_reg_evex(vec dst, vec a, vec b, vec atmp, vec btmp, kReg ktmp) %{
20455 predicate(!VM_Version::supports_avx10_2() && Matcher::vector_length_in_bytes(n) == 64 &&
20456 is_floating_point_type(Matcher::vector_element_basic_type(n))); // T_FLOAT, T_DOUBLE
20457 match(Set dst (MinV a b));
20458 match(Set dst (MaxV a b));
20459 effect(TEMP dst, USE a, USE b, TEMP atmp, TEMP btmp, TEMP ktmp);
20460 format %{ "vector_minmaxFP $dst,$a,$b\t!using $atmp, $btmp as TEMP" %}
20461 ins_encode %{
20462 assert(UseAVX > 2, "required");
20463
20464 int opcode = this->ideal_Opcode();
20465 int vlen_enc = vector_length_encoding(this);
20466 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20467
20468 __ evminmax_fp(opcode, elem_bt,
20469 $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister,
20470 $ktmp$$KRegister, $atmp$$XMMRegister , $btmp$$XMMRegister, vlen_enc);
20471 %}
20472 ins_pipe( pipe_slow );
20473 %}
20474
20475 // ------------------------------ Unsigned vector Min/Max ----------------------
20476
20477 instruct vector_uminmax_reg(vec dst, vec a, vec b) %{
20478 predicate(VM_Version::supports_avx512vl() || Matcher::vector_element_basic_type(n) != T_LONG);
20479 match(Set dst (UMinV a b));
20480 match(Set dst (UMaxV a b));
20481 format %{ "vector_uminmax $dst,$a,$b\t!" %}
20482 ins_encode %{
20483 int opcode = this->ideal_Opcode();
20484 int vlen_enc = vector_length_encoding(this);
20485 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20486 assert(is_integral_type(elem_bt), "");
20487 __ vpuminmax(opcode, elem_bt, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vlen_enc);
20488 %}
20489 ins_pipe( pipe_slow );
20490 %}
20491
20492 instruct vector_uminmax_mem(vec dst, vec a, memory b) %{
20493 predicate(VM_Version::supports_avx512vl() || Matcher::vector_element_basic_type(n) != T_LONG);
20494 match(Set dst (UMinV a (LoadVector b)));
20495 match(Set dst (UMaxV a (LoadVector b)));
20496 format %{ "vector_uminmax $dst,$a,$b\t!" %}
20497 ins_encode %{
20498 int opcode = this->ideal_Opcode();
20499 int vlen_enc = vector_length_encoding(this);
20500 BasicType elem_bt = Matcher::vector_element_basic_type(this);
20501 assert(is_integral_type(elem_bt), "");
20502 __ vpuminmax(opcode, elem_bt, $dst$$XMMRegister, $a$$XMMRegister, $b$$Address, vlen_enc);
20503 %}
20504 ins_pipe( pipe_slow );
20505 %}
20506
20507 instruct vector_uminmaxq_reg(vec dst, vec a, vec b, vec xtmp1, vec xtmp2) %{
20508 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_element_basic_type(n) == T_LONG);
20509 match(Set dst (UMinV a b));
20510 match(Set dst (UMaxV a b));
20511 effect(TEMP xtmp1, TEMP xtmp2);
20512 format %{ "vector_uminmaxq $dst,$a,$b\t! using xtmp1 and xtmp2 as TEMP" %}
20513 ins_encode %{
20514 int opcode = this->ideal_Opcode();
20515 int vlen_enc = vector_length_encoding(this);
20516 __ vpuminmaxq(opcode, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
20517 %}
20518 ins_pipe( pipe_slow );
20519 %}
20520
20521 instruct vector_uminmax_reg_masked(vec dst, vec src2, kReg mask) %{
20522 match(Set dst (UMinV (Binary dst src2) mask));
20523 match(Set dst (UMaxV (Binary dst src2) mask));
20524 format %{ "vector_uminmax_masked $dst, $dst, $src2, $mask\t! umin/max masked operation" %}
20525 ins_encode %{
20526 int vlen_enc = vector_length_encoding(this);
20527 BasicType bt = Matcher::vector_element_basic_type(this);
20528 int opc = this->ideal_Opcode();
20529 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
20530 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
20531 %}
20532 ins_pipe( pipe_slow );
20533 %}
20534
20535 instruct vector_uminmax_mem_masked(vec dst, memory src2, kReg mask) %{
20536 match(Set dst (UMinV (Binary dst (LoadVector src2)) mask));
20537 match(Set dst (UMaxV (Binary dst (LoadVector src2)) mask));
20538 format %{ "vector_uminmax_masked $dst, $dst, $src2, $mask\t! umin/max masked operation" %}
20539 ins_encode %{
20540 int vlen_enc = vector_length_encoding(this);
20541 BasicType bt = Matcher::vector_element_basic_type(this);
20542 int opc = this->ideal_Opcode();
20543 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
20544 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
20545 %}
20546 ins_pipe( pipe_slow );
20547 %}
20548
20549 // --------------------------------- Signum/CopySign ---------------------------
20550
20551 instruct signumF_reg(regF dst, regF zero, regF one, rFlagsReg cr) %{
20552 match(Set dst (SignumF dst (Binary zero one)));
20553 effect(KILL cr);
20554 format %{ "signumF $dst, $dst" %}
20555 ins_encode %{
20556 int opcode = this->ideal_Opcode();
20557 __ signum_fp(opcode, $dst$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister);
20558 %}
20559 ins_pipe( pipe_slow );
20560 %}
20561
20562 instruct signumD_reg(regD dst, regD zero, regD one, rFlagsReg cr) %{
20563 match(Set dst (SignumD dst (Binary zero one)));
20564 effect(KILL cr);
20565 format %{ "signumD $dst, $dst" %}
20566 ins_encode %{
20567 int opcode = this->ideal_Opcode();
20568 __ signum_fp(opcode, $dst$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister);
20569 %}
20570 ins_pipe( pipe_slow );
20571 %}
20572
20573 instruct signumV_reg_avx(vec dst, vec src, vec zero, vec one, vec xtmp1) %{
20574 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n) <= 32);
20575 match(Set dst (SignumVF src (Binary zero one)));
20576 match(Set dst (SignumVD src (Binary zero one)));
20577 effect(TEMP dst, TEMP xtmp1);
20578 format %{ "vector_signum_avx $dst, $src\t! using $xtmp1 as TEMP" %}
20579 ins_encode %{
20580 int opcode = this->ideal_Opcode();
20581 int vec_enc = vector_length_encoding(this);
20582 __ vector_signum_avx(opcode, $dst$$XMMRegister, $src$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister,
20583 $xtmp1$$XMMRegister, vec_enc);
20584 %}
20585 ins_pipe( pipe_slow );
20586 %}
20587
20588 instruct signumV_reg_evex(vec dst, vec src, vec zero, vec one, kReg ktmp1) %{
20589 predicate(VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64);
20590 match(Set dst (SignumVF src (Binary zero one)));
20591 match(Set dst (SignumVD src (Binary zero one)));
20592 effect(TEMP dst, TEMP ktmp1);
20593 format %{ "vector_signum_evex $dst, $src\t! using $ktmp1 as TEMP" %}
20594 ins_encode %{
20595 int opcode = this->ideal_Opcode();
20596 int vec_enc = vector_length_encoding(this);
20597 __ vector_signum_evex(opcode, $dst$$XMMRegister, $src$$XMMRegister, $zero$$XMMRegister, $one$$XMMRegister,
20598 $ktmp1$$KRegister, vec_enc);
20599 %}
20600 ins_pipe( pipe_slow );
20601 %}
20602
20603 // ---------------------------------------
20604 // For copySign use 0xE4 as writemask for vpternlog
20605 // Desired Truth Table: A -> xmm0 bit, B -> xmm1 bit, C -> xmm2 bit
20606 // C (xmm2) is set to 0x7FFFFFFF
20607 // Wherever xmm2 is 0, we want to pick from B (sign)
20608 // Wherever xmm2 is 1, we want to pick from A (src)
20609 //
20610 // A B C Result
20611 // 0 0 0 0
20612 // 0 0 1 0
20613 // 0 1 0 1
20614 // 0 1 1 0
20615 // 1 0 0 0
20616 // 1 0 1 1
20617 // 1 1 0 1
20618 // 1 1 1 1
20619 //
20620 // Result going from high bit to low bit is 0x11100100 = 0xe4
20621 // ---------------------------------------
20622
20623 instruct copySignF_reg(regF dst, regF src, regF tmp1, rRegI tmp2) %{
20624 match(Set dst (CopySignF dst src));
20625 effect(TEMP tmp1, TEMP tmp2);
20626 format %{ "CopySignF $dst, $src\t! using $tmp1 and $tmp2 as TEMP" %}
20627 ins_encode %{
20628 __ movl($tmp2$$Register, 0x7FFFFFFF);
20629 __ movdl($tmp1$$XMMRegister, $tmp2$$Register);
20630 __ vpternlogd($dst$$XMMRegister, 0xE4, $src$$XMMRegister, $tmp1$$XMMRegister, Assembler::AVX_128bit);
20631 %}
20632 ins_pipe( pipe_slow );
20633 %}
20634
20635 instruct copySignD_imm(regD dst, regD src, regD tmp1, rRegL tmp2, immD zero) %{
20636 match(Set dst (CopySignD dst (Binary src zero)));
20637 ins_cost(100);
20638 effect(TEMP tmp1, TEMP tmp2);
20639 format %{ "CopySignD $dst, $src\t! using $tmp1 and $tmp2 as TEMP" %}
20640 ins_encode %{
20641 __ mov64($tmp2$$Register, 0x7FFFFFFFFFFFFFFF);
20642 __ movq($tmp1$$XMMRegister, $tmp2$$Register);
20643 __ vpternlogq($dst$$XMMRegister, 0xE4, $src$$XMMRegister, $tmp1$$XMMRegister, Assembler::AVX_128bit);
20644 %}
20645 ins_pipe( pipe_slow );
20646 %}
20647
20648 //----------------------------- CompressBits/ExpandBits ------------------------
20649
20650 instruct compressBitsI_reg(rRegI dst, rRegI src, rRegI mask) %{
20651 predicate(n->bottom_type()->isa_int());
20652 match(Set dst (CompressBits src mask));
20653 format %{ "pextl $dst, $src, $mask\t! parallel bit extract" %}
20654 ins_encode %{
20655 __ pextl($dst$$Register, $src$$Register, $mask$$Register);
20656 %}
20657 ins_pipe( pipe_slow );
20658 %}
20659
20660 instruct expandBitsI_reg(rRegI dst, rRegI src, rRegI mask) %{
20661 predicate(n->bottom_type()->isa_int());
20662 match(Set dst (ExpandBits src mask));
20663 format %{ "pdepl $dst, $src, $mask\t! parallel bit deposit" %}
20664 ins_encode %{
20665 __ pdepl($dst$$Register, $src$$Register, $mask$$Register);
20666 %}
20667 ins_pipe( pipe_slow );
20668 %}
20669
20670 instruct compressBitsI_mem(rRegI dst, rRegI src, memory mask) %{
20671 predicate(n->bottom_type()->isa_int());
20672 match(Set dst (CompressBits src (LoadI mask)));
20673 format %{ "pextl $dst, $src, $mask\t! parallel bit extract" %}
20674 ins_encode %{
20675 __ pextl($dst$$Register, $src$$Register, $mask$$Address);
20676 %}
20677 ins_pipe( pipe_slow );
20678 %}
20679
20680 instruct expandBitsI_mem(rRegI dst, rRegI src, memory mask) %{
20681 predicate(n->bottom_type()->isa_int());
20682 match(Set dst (ExpandBits src (LoadI mask)));
20683 format %{ "pdepl $dst, $src, $mask\t! parallel bit deposit" %}
20684 ins_encode %{
20685 __ pdepl($dst$$Register, $src$$Register, $mask$$Address);
20686 %}
20687 ins_pipe( pipe_slow );
20688 %}
20689
20690 // --------------------------------- Sqrt --------------------------------------
20691
20692 instruct vsqrtF_reg(vec dst, vec src) %{
20693 match(Set dst (SqrtVF src));
20694 format %{ "vsqrtps $dst,$src\t! sqrt packedF" %}
20695 ins_encode %{
20696 assert(UseAVX > 0, "required");
20697 int vlen_enc = vector_length_encoding(this);
20698 __ vsqrtps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
20699 %}
20700 ins_pipe( pipe_slow );
20701 %}
20702
20703 instruct vsqrtF_mem(vec dst, memory mem) %{
20704 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
20705 match(Set dst (SqrtVF (LoadVector mem)));
20706 format %{ "vsqrtps $dst,$mem\t! sqrt packedF" %}
20707 ins_encode %{
20708 assert(UseAVX > 0, "required");
20709 int vlen_enc = vector_length_encoding(this);
20710 __ vsqrtps($dst$$XMMRegister, $mem$$Address, vlen_enc);
20711 %}
20712 ins_pipe( pipe_slow );
20713 %}
20714
20715 // Floating point vector sqrt
20716 instruct vsqrtD_reg(vec dst, vec src) %{
20717 match(Set dst (SqrtVD src));
20718 format %{ "vsqrtpd $dst,$src\t! sqrt packedD" %}
20719 ins_encode %{
20720 assert(UseAVX > 0, "required");
20721 int vlen_enc = vector_length_encoding(this);
20722 __ vsqrtpd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
20723 %}
20724 ins_pipe( pipe_slow );
20725 %}
20726
20727 instruct vsqrtD_mem(vec dst, memory mem) %{
20728 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
20729 match(Set dst (SqrtVD (LoadVector mem)));
20730 format %{ "vsqrtpd $dst,$mem\t! sqrt packedD" %}
20731 ins_encode %{
20732 assert(UseAVX > 0, "required");
20733 int vlen_enc = vector_length_encoding(this);
20734 __ vsqrtpd($dst$$XMMRegister, $mem$$Address, vlen_enc);
20735 %}
20736 ins_pipe( pipe_slow );
20737 %}
20738
20739 // ------------------------------ Shift ---------------------------------------
20740
20741 // Left and right shift count vectors are the same on x86
20742 // (only lowest bits of xmm reg are used for count).
20743 instruct vshiftcnt(vec dst, rRegI cnt) %{
20744 match(Set dst (LShiftCntV cnt));
20745 match(Set dst (RShiftCntV cnt));
20746 format %{ "movdl $dst,$cnt\t! load shift count" %}
20747 ins_encode %{
20748 __ movdl($dst$$XMMRegister, $cnt$$Register);
20749 %}
20750 ins_pipe( pipe_slow );
20751 %}
20752
20753 // Byte vector shift
20754 instruct vshiftB(vec dst, vec src, vec shift, vec tmp) %{
20755 predicate(Matcher::vector_length(n) <= 8 && !n->as_ShiftV()->is_var_shift());
20756 match(Set dst ( LShiftVB src shift));
20757 match(Set dst ( RShiftVB src shift));
20758 match(Set dst (URShiftVB src shift));
20759 effect(TEMP dst, USE src, USE shift, TEMP tmp);
20760 format %{"vector_byte_shift $dst,$src,$shift" %}
20761 ins_encode %{
20762 assert(UseSSE > 3, "required");
20763 int opcode = this->ideal_Opcode();
20764 bool sign = (opcode != Op_URShiftVB);
20765 __ vextendbw(sign, $tmp$$XMMRegister, $src$$XMMRegister);
20766 __ vshiftw(opcode, $tmp$$XMMRegister, $shift$$XMMRegister);
20767 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), noreg);
20768 __ pand($dst$$XMMRegister, $tmp$$XMMRegister);
20769 __ packuswb($dst$$XMMRegister, $dst$$XMMRegister);
20770 %}
20771 ins_pipe( pipe_slow );
20772 %}
20773
20774 instruct vshift16B(vec dst, vec src, vec shift, vec tmp1, vec tmp2) %{
20775 predicate(Matcher::vector_length(n) == 16 && !n->as_ShiftV()->is_var_shift() &&
20776 UseAVX <= 1);
20777 match(Set dst ( LShiftVB src shift));
20778 match(Set dst ( RShiftVB src shift));
20779 match(Set dst (URShiftVB src shift));
20780 effect(TEMP dst, USE src, USE shift, TEMP tmp1, TEMP tmp2);
20781 format %{"vector_byte_shift $dst,$src,$shift" %}
20782 ins_encode %{
20783 assert(UseSSE > 3, "required");
20784 int opcode = this->ideal_Opcode();
20785 bool sign = (opcode != Op_URShiftVB);
20786 __ vextendbw(sign, $tmp1$$XMMRegister, $src$$XMMRegister);
20787 __ vshiftw(opcode, $tmp1$$XMMRegister, $shift$$XMMRegister);
20788 __ pshufd($tmp2$$XMMRegister, $src$$XMMRegister, 0xE);
20789 __ vextendbw(sign, $tmp2$$XMMRegister, $tmp2$$XMMRegister);
20790 __ vshiftw(opcode, $tmp2$$XMMRegister, $shift$$XMMRegister);
20791 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), noreg);
20792 __ pand($tmp2$$XMMRegister, $dst$$XMMRegister);
20793 __ pand($dst$$XMMRegister, $tmp1$$XMMRegister);
20794 __ packuswb($dst$$XMMRegister, $tmp2$$XMMRegister);
20795 %}
20796 ins_pipe( pipe_slow );
20797 %}
20798
20799 instruct vshift16B_avx(vec dst, vec src, vec shift, vec tmp) %{
20800 predicate(Matcher::vector_length(n) == 16 && !n->as_ShiftV()->is_var_shift() &&
20801 UseAVX > 1);
20802 match(Set dst ( LShiftVB src shift));
20803 match(Set dst ( RShiftVB src shift));
20804 match(Set dst (URShiftVB src shift));
20805 effect(TEMP dst, TEMP tmp);
20806 format %{"vector_byte_shift $dst,$src,$shift" %}
20807 ins_encode %{
20808 int opcode = this->ideal_Opcode();
20809 bool sign = (opcode != Op_URShiftVB);
20810 int vlen_enc = Assembler::AVX_256bit;
20811 __ vextendbw(sign, $tmp$$XMMRegister, $src$$XMMRegister, vlen_enc);
20812 __ vshiftw(opcode, $tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20813 __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
20814 __ vextracti128_high($dst$$XMMRegister, $tmp$$XMMRegister);
20815 __ vpackuswb($dst$$XMMRegister, $tmp$$XMMRegister, $dst$$XMMRegister, 0);
20816 %}
20817 ins_pipe( pipe_slow );
20818 %}
20819
20820 instruct vshift32B_avx(vec dst, vec src, vec shift, vec tmp) %{
20821 predicate(Matcher::vector_length(n) == 32 && !n->as_ShiftV()->is_var_shift());
20822 match(Set dst ( LShiftVB src shift));
20823 match(Set dst ( RShiftVB src shift));
20824 match(Set dst (URShiftVB src shift));
20825 effect(TEMP dst, TEMP tmp);
20826 format %{"vector_byte_shift $dst,$src,$shift" %}
20827 ins_encode %{
20828 assert(UseAVX > 1, "required");
20829 int opcode = this->ideal_Opcode();
20830 bool sign = (opcode != Op_URShiftVB);
20831 int vlen_enc = Assembler::AVX_256bit;
20832 __ vextracti128_high($tmp$$XMMRegister, $src$$XMMRegister);
20833 __ vextendbw(sign, $tmp$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
20834 __ vextendbw(sign, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
20835 __ vshiftw(opcode, $tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20836 __ vshiftw(opcode, $dst$$XMMRegister, $dst$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20837 __ vpand($tmp$$XMMRegister, $tmp$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
20838 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
20839 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
20840 __ vpermq($dst$$XMMRegister, $dst$$XMMRegister, 0xD8, vlen_enc);
20841 %}
20842 ins_pipe( pipe_slow );
20843 %}
20844
20845 instruct vshift64B_avx(vec dst, vec src, vec shift, vec tmp1, vec tmp2) %{
20846 predicate(Matcher::vector_length(n) == 64 && !n->as_ShiftV()->is_var_shift());
20847 match(Set dst ( LShiftVB src shift));
20848 match(Set dst (RShiftVB src shift));
20849 match(Set dst (URShiftVB src shift));
20850 effect(TEMP dst, TEMP tmp1, TEMP tmp2);
20851 format %{"vector_byte_shift $dst,$src,$shift" %}
20852 ins_encode %{
20853 assert(UseAVX > 2, "required");
20854 int opcode = this->ideal_Opcode();
20855 bool sign = (opcode != Op_URShiftVB);
20856 int vlen_enc = Assembler::AVX_512bit;
20857 __ vextracti64x4($tmp1$$XMMRegister, $src$$XMMRegister, 1);
20858 __ vextendbw(sign, $tmp1$$XMMRegister, $tmp1$$XMMRegister, vlen_enc);
20859 __ vextendbw(sign, $tmp2$$XMMRegister, $src$$XMMRegister, vlen_enc);
20860 __ vshiftw(opcode, $tmp1$$XMMRegister, $tmp1$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20861 __ vshiftw(opcode, $tmp2$$XMMRegister, $tmp2$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20862 __ vmovdqu($dst$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), noreg);
20863 __ vpbroadcastd($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
20864 __ vpand($tmp1$$XMMRegister, $tmp1$$XMMRegister, $dst$$XMMRegister, vlen_enc);
20865 __ vpand($tmp2$$XMMRegister, $tmp2$$XMMRegister, $dst$$XMMRegister, vlen_enc);
20866 __ vpackuswb($dst$$XMMRegister, $tmp1$$XMMRegister, $tmp2$$XMMRegister, vlen_enc);
20867 __ evmovdquq($tmp2$$XMMRegister, ExternalAddress(vector_byte_perm_mask()), vlen_enc, noreg);
20868 __ vpermq($dst$$XMMRegister, $tmp2$$XMMRegister, $dst$$XMMRegister, vlen_enc);
20869 %}
20870 ins_pipe( pipe_slow );
20871 %}
20872
20873 // Shorts vector logical right shift produces incorrect Java result
20874 // for negative data because java code convert short value into int with
20875 // sign extension before a shift. But char vectors are fine since chars are
20876 // unsigned values.
20877 // Shorts/Chars vector left shift
20878 instruct vshiftS(vec dst, vec src, vec shift) %{
20879 predicate(!n->as_ShiftV()->is_var_shift());
20880 match(Set dst ( LShiftVS src shift));
20881 match(Set dst ( RShiftVS src shift));
20882 match(Set dst (URShiftVS src shift));
20883 effect(TEMP dst, USE src, USE shift);
20884 format %{ "vshiftw $dst,$src,$shift\t! shift packedS" %}
20885 ins_encode %{
20886 int opcode = this->ideal_Opcode();
20887 if (UseAVX > 0) {
20888 int vlen_enc = vector_length_encoding(this);
20889 __ vshiftw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20890 } else {
20891 int vlen = Matcher::vector_length(this);
20892 if (vlen == 2) {
20893 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
20894 __ vshiftw(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
20895 } else if (vlen == 4) {
20896 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
20897 __ vshiftw(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
20898 } else {
20899 assert (vlen == 8, "sanity");
20900 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
20901 __ vshiftw(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
20902 }
20903 }
20904 %}
20905 ins_pipe( pipe_slow );
20906 %}
20907
20908 // Integers vector left shift
20909 instruct vshiftI(vec dst, vec src, vec shift) %{
20910 predicate(!n->as_ShiftV()->is_var_shift());
20911 match(Set dst ( LShiftVI src shift));
20912 match(Set dst ( RShiftVI src shift));
20913 match(Set dst (URShiftVI src shift));
20914 effect(TEMP dst, USE src, USE shift);
20915 format %{ "vshiftd $dst,$src,$shift\t! shift packedI" %}
20916 ins_encode %{
20917 int opcode = this->ideal_Opcode();
20918 if (UseAVX > 0) {
20919 int vlen_enc = vector_length_encoding(this);
20920 __ vshiftd(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20921 } else {
20922 int vlen = Matcher::vector_length(this);
20923 if (vlen == 2) {
20924 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
20925 __ vshiftd(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
20926 } else {
20927 assert(vlen == 4, "sanity");
20928 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
20929 __ vshiftd(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
20930 }
20931 }
20932 %}
20933 ins_pipe( pipe_slow );
20934 %}
20935
20936 // Integers vector left constant shift
20937 instruct vshiftI_imm(vec dst, vec src, immI8 shift) %{
20938 match(Set dst (LShiftVI src (LShiftCntV shift)));
20939 match(Set dst (RShiftVI src (RShiftCntV shift)));
20940 match(Set dst (URShiftVI src (RShiftCntV shift)));
20941 format %{ "vshiftd_imm $dst,$src,$shift\t! shift packedI" %}
20942 ins_encode %{
20943 int opcode = this->ideal_Opcode();
20944 if (UseAVX > 0) {
20945 int vector_len = vector_length_encoding(this);
20946 __ vshiftd_imm(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$constant, vector_len);
20947 } else {
20948 int vlen = Matcher::vector_length(this);
20949 if (vlen == 2) {
20950 __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
20951 __ vshiftd_imm(opcode, $dst$$XMMRegister, $shift$$constant);
20952 } else {
20953 assert(vlen == 4, "sanity");
20954 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
20955 __ vshiftd_imm(opcode, $dst$$XMMRegister, $shift$$constant);
20956 }
20957 }
20958 %}
20959 ins_pipe( pipe_slow );
20960 %}
20961
20962 // Longs vector shift
20963 instruct vshiftL(vec dst, vec src, vec shift) %{
20964 predicate(!n->as_ShiftV()->is_var_shift());
20965 match(Set dst ( LShiftVL src shift));
20966 match(Set dst (URShiftVL src shift));
20967 effect(TEMP dst, USE src, USE shift);
20968 format %{ "vshiftq $dst,$src,$shift\t! shift packedL" %}
20969 ins_encode %{
20970 int opcode = this->ideal_Opcode();
20971 if (UseAVX > 0) {
20972 int vlen_enc = vector_length_encoding(this);
20973 __ vshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
20974 } else {
20975 assert(Matcher::vector_length(this) == 2, "");
20976 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
20977 __ vshiftq(opcode, $dst$$XMMRegister, $shift$$XMMRegister);
20978 }
20979 %}
20980 ins_pipe( pipe_slow );
20981 %}
20982
20983 // Longs vector constant shift
20984 instruct vshiftL_imm(vec dst, vec src, immI8 shift) %{
20985 match(Set dst (LShiftVL src (LShiftCntV shift)));
20986 match(Set dst (URShiftVL src (RShiftCntV shift)));
20987 format %{ "vshiftq_imm $dst,$src,$shift\t! shift packedL" %}
20988 ins_encode %{
20989 int opcode = this->ideal_Opcode();
20990 if (UseAVX > 0) {
20991 int vector_len = vector_length_encoding(this);
20992 __ vshiftq_imm(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$constant, vector_len);
20993 } else {
20994 assert(Matcher::vector_length(this) == 2, "");
20995 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
20996 __ vshiftq_imm(opcode, $dst$$XMMRegister, $shift$$constant);
20997 }
20998 %}
20999 ins_pipe( pipe_slow );
21000 %}
21001
21002 // -------------------ArithmeticRightShift -----------------------------------
21003 // Long vector arithmetic right shift
21004 instruct vshiftL_arith_reg(vec dst, vec src, vec shift, vec tmp) %{
21005 predicate(!n->as_ShiftV()->is_var_shift() && UseAVX <= 2);
21006 match(Set dst (RShiftVL src shift));
21007 effect(TEMP dst, TEMP tmp);
21008 format %{ "vshiftq $dst,$src,$shift" %}
21009 ins_encode %{
21010 uint vlen = Matcher::vector_length(this);
21011 if (vlen == 2) {
21012 __ movdqu($dst$$XMMRegister, $src$$XMMRegister);
21013 __ psrlq($dst$$XMMRegister, $shift$$XMMRegister);
21014 __ movdqu($tmp$$XMMRegister, ExternalAddress(vector_long_sign_mask()), noreg);
21015 __ psrlq($tmp$$XMMRegister, $shift$$XMMRegister);
21016 __ pxor($dst$$XMMRegister, $tmp$$XMMRegister);
21017 __ psubq($dst$$XMMRegister, $tmp$$XMMRegister);
21018 } else {
21019 assert(vlen == 4, "sanity");
21020 assert(UseAVX > 1, "required");
21021 int vlen_enc = Assembler::AVX_256bit;
21022 __ vpsrlq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21023 __ vmovdqu($tmp$$XMMRegister, ExternalAddress(vector_long_sign_mask()), noreg);
21024 __ vpsrlq($tmp$$XMMRegister, $tmp$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21025 __ vpxor($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
21026 __ vpsubq($dst$$XMMRegister, $dst$$XMMRegister, $tmp$$XMMRegister, vlen_enc);
21027 }
21028 %}
21029 ins_pipe( pipe_slow );
21030 %}
21031
21032 instruct vshiftL_arith_reg_evex(vec dst, vec src, vec shift) %{
21033 predicate(!n->as_ShiftV()->is_var_shift() && UseAVX > 2);
21034 match(Set dst (RShiftVL src shift));
21035 format %{ "vshiftq $dst,$src,$shift" %}
21036 ins_encode %{
21037 int vlen_enc = vector_length_encoding(this);
21038 __ evpsraq($dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21039 %}
21040 ins_pipe( pipe_slow );
21041 %}
21042
21043 // ------------------- Variable Shift -----------------------------
21044 // Byte variable shift
21045 instruct vshift8B_var_nobw(vec dst, vec src, vec shift, vec vtmp) %{
21046 predicate(Matcher::vector_length(n) <= 8 &&
21047 n->as_ShiftV()->is_var_shift() &&
21048 !VM_Version::supports_avx512bw());
21049 match(Set dst ( LShiftVB src shift));
21050 match(Set dst ( RShiftVB src shift));
21051 match(Set dst (URShiftVB src shift));
21052 effect(TEMP dst, TEMP vtmp);
21053 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp as TEMP" %}
21054 ins_encode %{
21055 assert(UseAVX >= 2, "required");
21056
21057 int opcode = this->ideal_Opcode();
21058 int vlen_enc = Assembler::AVX_128bit;
21059 __ varshiftbw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp$$XMMRegister);
21060 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, 0);
21061 %}
21062 ins_pipe( pipe_slow );
21063 %}
21064
21065 instruct vshift16B_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2) %{
21066 predicate(Matcher::vector_length(n) == 16 &&
21067 n->as_ShiftV()->is_var_shift() &&
21068 !VM_Version::supports_avx512bw());
21069 match(Set dst ( LShiftVB src shift));
21070 match(Set dst ( RShiftVB src shift));
21071 match(Set dst (URShiftVB src shift));
21072 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
21073 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp1, $vtmp2 as TEMP" %}
21074 ins_encode %{
21075 assert(UseAVX >= 2, "required");
21076
21077 int opcode = this->ideal_Opcode();
21078 int vlen_enc = Assembler::AVX_128bit;
21079 // Shift lower half and get word result in dst
21080 __ varshiftbw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp1$$XMMRegister);
21081
21082 // Shift upper half and get word result in vtmp1
21083 __ vpshufd($vtmp1$$XMMRegister, $src$$XMMRegister, 0xE, 0);
21084 __ vpshufd($vtmp2$$XMMRegister, $shift$$XMMRegister, 0xE, 0);
21085 __ varshiftbw(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21086
21087 // Merge and down convert the two word results to byte in dst
21088 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0);
21089 %}
21090 ins_pipe( pipe_slow );
21091 %}
21092
21093 instruct vshift32B_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2, vec vtmp3, vec vtmp4) %{
21094 predicate(Matcher::vector_length(n) == 32 &&
21095 n->as_ShiftV()->is_var_shift() &&
21096 !VM_Version::supports_avx512bw());
21097 match(Set dst ( LShiftVB src shift));
21098 match(Set dst ( RShiftVB src shift));
21099 match(Set dst (URShiftVB src shift));
21100 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4);
21101 format %{ "vector_varshift_byte $dst, $src, $shift\n\t using $vtmp1, $vtmp2, $vtmp3, $vtmp4 as TEMP" %}
21102 ins_encode %{
21103 assert(UseAVX >= 2, "required");
21104
21105 int opcode = this->ideal_Opcode();
21106 int vlen_enc = Assembler::AVX_128bit;
21107 // Process lower 128 bits and get result in dst
21108 __ varshiftbw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp1$$XMMRegister);
21109 __ vpshufd($vtmp1$$XMMRegister, $src$$XMMRegister, 0xE, 0);
21110 __ vpshufd($vtmp2$$XMMRegister, $shift$$XMMRegister, 0xE, 0);
21111 __ varshiftbw(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21112 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0);
21113
21114 // Process higher 128 bits and get result in vtmp3
21115 __ vextracti128_high($vtmp1$$XMMRegister, $src$$XMMRegister);
21116 __ vextracti128_high($vtmp2$$XMMRegister, $shift$$XMMRegister);
21117 __ varshiftbw(opcode, $vtmp3$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp4$$XMMRegister);
21118 __ vpshufd($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, 0xE, 0);
21119 __ vpshufd($vtmp2$$XMMRegister, $vtmp2$$XMMRegister, 0xE, 0);
21120 __ varshiftbw(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21121 __ vpackuswb($vtmp1$$XMMRegister, $vtmp3$$XMMRegister, $vtmp1$$XMMRegister, 0);
21122
21123 // Merge the two results in dst
21124 __ vinserti128($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0x1);
21125 %}
21126 ins_pipe( pipe_slow );
21127 %}
21128
21129 instruct vshiftB_var_evex_bw(vec dst, vec src, vec shift, vec vtmp) %{
21130 predicate(Matcher::vector_length(n) <= 32 &&
21131 n->as_ShiftV()->is_var_shift() &&
21132 VM_Version::supports_avx512bw());
21133 match(Set dst ( LShiftVB src shift));
21134 match(Set dst ( RShiftVB src shift));
21135 match(Set dst (URShiftVB src shift));
21136 effect(TEMP dst, TEMP vtmp);
21137 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp as TEMP" %}
21138 ins_encode %{
21139 assert(UseAVX > 2, "required");
21140
21141 int opcode = this->ideal_Opcode();
21142 int vlen_enc = vector_length_encoding(this);
21143 __ evarshiftb(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp$$XMMRegister);
21144 %}
21145 ins_pipe( pipe_slow );
21146 %}
21147
21148 instruct vshift64B_var_evex_bw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2) %{
21149 predicate(Matcher::vector_length(n) == 64 &&
21150 n->as_ShiftV()->is_var_shift() &&
21151 VM_Version::supports_avx512bw());
21152 match(Set dst ( LShiftVB src shift));
21153 match(Set dst ( RShiftVB src shift));
21154 match(Set dst (URShiftVB src shift));
21155 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
21156 format %{ "vector_varshift_byte $dst, $src, $shift\n\t! using $vtmp1, $vtmp2 as TEMP" %}
21157 ins_encode %{
21158 assert(UseAVX > 2, "required");
21159
21160 int opcode = this->ideal_Opcode();
21161 int vlen_enc = Assembler::AVX_256bit;
21162 __ evarshiftb(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc, $vtmp1$$XMMRegister);
21163 __ vextracti64x4_high($vtmp1$$XMMRegister, $src$$XMMRegister);
21164 __ vextracti64x4_high($vtmp2$$XMMRegister, $shift$$XMMRegister);
21165 __ evarshiftb(opcode, $vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, vlen_enc, $vtmp2$$XMMRegister);
21166 __ vinserti64x4($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, 0x1);
21167 %}
21168 ins_pipe( pipe_slow );
21169 %}
21170
21171 // Short variable shift
21172 instruct vshift8S_var_nobw(vec dst, vec src, vec shift, vec vtmp) %{
21173 predicate(Matcher::vector_length(n) <= 8 &&
21174 n->as_ShiftV()->is_var_shift() &&
21175 !VM_Version::supports_avx512bw());
21176 match(Set dst ( LShiftVS src shift));
21177 match(Set dst ( RShiftVS src shift));
21178 match(Set dst (URShiftVS src shift));
21179 effect(TEMP dst, TEMP vtmp);
21180 format %{ "vector_var_shift_left_short $dst, $src, $shift\n\t" %}
21181 ins_encode %{
21182 assert(UseAVX >= 2, "required");
21183
21184 int opcode = this->ideal_Opcode();
21185 bool sign = (opcode != Op_URShiftVS);
21186 int vlen_enc = Assembler::AVX_256bit;
21187 __ vextendwd(sign, $dst$$XMMRegister, $src$$XMMRegister, 1);
21188 __ vpmovzxwd($vtmp$$XMMRegister, $shift$$XMMRegister, 1);
21189 __ varshiftd(opcode, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
21190 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21191 __ vextracti128_high($vtmp$$XMMRegister, $dst$$XMMRegister);
21192 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0);
21193 %}
21194 ins_pipe( pipe_slow );
21195 %}
21196
21197 instruct vshift16S_var_nobw(vec dst, vec src, vec shift, vec vtmp1, vec vtmp2) %{
21198 predicate(Matcher::vector_length(n) == 16 &&
21199 n->as_ShiftV()->is_var_shift() &&
21200 !VM_Version::supports_avx512bw());
21201 match(Set dst ( LShiftVS src shift));
21202 match(Set dst ( RShiftVS src shift));
21203 match(Set dst (URShiftVS src shift));
21204 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
21205 format %{ "vector_var_shift_left_short $dst, $src, $shift\n\t" %}
21206 ins_encode %{
21207 assert(UseAVX >= 2, "required");
21208
21209 int opcode = this->ideal_Opcode();
21210 bool sign = (opcode != Op_URShiftVS);
21211 int vlen_enc = Assembler::AVX_256bit;
21212 // Shift lower half, with result in vtmp2 using vtmp1 as TEMP
21213 __ vextendwd(sign, $vtmp2$$XMMRegister, $src$$XMMRegister, vlen_enc);
21214 __ vpmovzxwd($vtmp1$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21215 __ varshiftd(opcode, $vtmp2$$XMMRegister, $vtmp2$$XMMRegister, $vtmp1$$XMMRegister, vlen_enc);
21216 __ vpand($vtmp2$$XMMRegister, $vtmp2$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21217
21218 // Shift upper half, with result in dst using vtmp1 as TEMP
21219 __ vextracti128_high($dst$$XMMRegister, $src$$XMMRegister);
21220 __ vextracti128_high($vtmp1$$XMMRegister, $shift$$XMMRegister);
21221 __ vextendwd(sign, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21222 __ vpmovzxwd($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, vlen_enc);
21223 __ varshiftd(opcode, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, vlen_enc);
21224 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21225
21226 // Merge lower and upper half result into dst
21227 __ vpackusdw($dst$$XMMRegister, $vtmp2$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21228 __ vpermq($dst$$XMMRegister, $dst$$XMMRegister, 0xD8, vlen_enc);
21229 %}
21230 ins_pipe( pipe_slow );
21231 %}
21232
21233 instruct vshift16S_var_evex_bw(vec dst, vec src, vec shift) %{
21234 predicate(n->as_ShiftV()->is_var_shift() &&
21235 VM_Version::supports_avx512bw());
21236 match(Set dst ( LShiftVS src shift));
21237 match(Set dst ( RShiftVS src shift));
21238 match(Set dst (URShiftVS src shift));
21239 format %{ "vector_varshift_short $dst,$src,$shift\t!" %}
21240 ins_encode %{
21241 assert(UseAVX > 2, "required");
21242
21243 int opcode = this->ideal_Opcode();
21244 int vlen_enc = vector_length_encoding(this);
21245 if (!VM_Version::supports_avx512vl()) {
21246 vlen_enc = Assembler::AVX_512bit;
21247 }
21248 __ varshiftw(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21249 %}
21250 ins_pipe( pipe_slow );
21251 %}
21252
21253 //Integer variable shift
21254 instruct vshiftI_var(vec dst, vec src, vec shift) %{
21255 predicate(n->as_ShiftV()->is_var_shift());
21256 match(Set dst ( LShiftVI src shift));
21257 match(Set dst ( RShiftVI src shift));
21258 match(Set dst (URShiftVI src shift));
21259 format %{ "vector_varshift_int $dst,$src,$shift\t!" %}
21260 ins_encode %{
21261 assert(UseAVX >= 2, "required");
21262
21263 int opcode = this->ideal_Opcode();
21264 int vlen_enc = vector_length_encoding(this);
21265 __ varshiftd(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21266 %}
21267 ins_pipe( pipe_slow );
21268 %}
21269
21270 //Long variable shift
21271 instruct vshiftL_var(vec dst, vec src, vec shift) %{
21272 predicate(n->as_ShiftV()->is_var_shift());
21273 match(Set dst ( LShiftVL src shift));
21274 match(Set dst (URShiftVL src shift));
21275 format %{ "vector_varshift_long $dst,$src,$shift\t!" %}
21276 ins_encode %{
21277 assert(UseAVX >= 2, "required");
21278
21279 int opcode = this->ideal_Opcode();
21280 int vlen_enc = vector_length_encoding(this);
21281 __ varshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21282 %}
21283 ins_pipe( pipe_slow );
21284 %}
21285
21286 //Long variable right shift arithmetic
21287 instruct vshiftL_arith_var(vec dst, vec src, vec shift, vec vtmp) %{
21288 predicate(Matcher::vector_length(n) <= 4 &&
21289 n->as_ShiftV()->is_var_shift() &&
21290 UseAVX == 2);
21291 match(Set dst (RShiftVL src shift));
21292 effect(TEMP dst, TEMP vtmp);
21293 format %{ "vector_varshift_long $dst,$src,$shift\n\t! using $vtmp as TEMP" %}
21294 ins_encode %{
21295 int opcode = this->ideal_Opcode();
21296 int vlen_enc = vector_length_encoding(this);
21297 __ varshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc,
21298 $vtmp$$XMMRegister);
21299 %}
21300 ins_pipe( pipe_slow );
21301 %}
21302
21303 instruct vshiftL_arith_var_evex(vec dst, vec src, vec shift) %{
21304 predicate(n->as_ShiftV()->is_var_shift() &&
21305 UseAVX > 2);
21306 match(Set dst (RShiftVL src shift));
21307 format %{ "vector_varfshift_long $dst,$src,$shift\t!" %}
21308 ins_encode %{
21309 int opcode = this->ideal_Opcode();
21310 int vlen_enc = vector_length_encoding(this);
21311 __ varshiftq(opcode, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vlen_enc);
21312 %}
21313 ins_pipe( pipe_slow );
21314 %}
21315
21316 // --------------------------------- AND --------------------------------------
21317
21318 instruct vand(vec dst, vec src) %{
21319 predicate(UseAVX == 0);
21320 match(Set dst (AndV dst src));
21321 format %{ "pand $dst,$src\t! and vectors" %}
21322 ins_encode %{
21323 __ pand($dst$$XMMRegister, $src$$XMMRegister);
21324 %}
21325 ins_pipe( pipe_slow );
21326 %}
21327
21328 instruct vand_reg(vec dst, vec src1, vec src2) %{
21329 predicate(UseAVX > 0);
21330 match(Set dst (AndV src1 src2));
21331 format %{ "vpand $dst,$src1,$src2\t! and vectors" %}
21332 ins_encode %{
21333 int vlen_enc = vector_length_encoding(this);
21334 __ vpand($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
21335 %}
21336 ins_pipe( pipe_slow );
21337 %}
21338
21339 instruct vand_mem(vec dst, vec src, memory mem) %{
21340 predicate((UseAVX > 0) &&
21341 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
21342 match(Set dst (AndV src (LoadVector mem)));
21343 format %{ "vpand $dst,$src,$mem\t! and vectors" %}
21344 ins_encode %{
21345 int vlen_enc = vector_length_encoding(this);
21346 __ vpand($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
21347 %}
21348 ins_pipe( pipe_slow );
21349 %}
21350
21351 // --------------------------------- OR ---------------------------------------
21352
21353 instruct vor(vec dst, vec src) %{
21354 predicate(UseAVX == 0);
21355 match(Set dst (OrV dst src));
21356 format %{ "por $dst,$src\t! or vectors" %}
21357 ins_encode %{
21358 __ por($dst$$XMMRegister, $src$$XMMRegister);
21359 %}
21360 ins_pipe( pipe_slow );
21361 %}
21362
21363 instruct vor_reg(vec dst, vec src1, vec src2) %{
21364 predicate(UseAVX > 0);
21365 match(Set dst (OrV src1 src2));
21366 format %{ "vpor $dst,$src1,$src2\t! or vectors" %}
21367 ins_encode %{
21368 int vlen_enc = vector_length_encoding(this);
21369 __ vpor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
21370 %}
21371 ins_pipe( pipe_slow );
21372 %}
21373
21374 instruct vor_mem(vec dst, vec src, memory mem) %{
21375 predicate((UseAVX > 0) &&
21376 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
21377 match(Set dst (OrV src (LoadVector mem)));
21378 format %{ "vpor $dst,$src,$mem\t! or vectors" %}
21379 ins_encode %{
21380 int vlen_enc = vector_length_encoding(this);
21381 __ vpor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
21382 %}
21383 ins_pipe( pipe_slow );
21384 %}
21385
21386 // --------------------------------- XOR --------------------------------------
21387
21388 instruct vxor(vec dst, vec src) %{
21389 predicate(UseAVX == 0);
21390 match(Set dst (XorV dst src));
21391 format %{ "pxor $dst,$src\t! xor vectors" %}
21392 ins_encode %{
21393 __ pxor($dst$$XMMRegister, $src$$XMMRegister);
21394 %}
21395 ins_pipe( pipe_slow );
21396 %}
21397
21398 instruct vxor_reg(vec dst, vec src1, vec src2) %{
21399 predicate(UseAVX > 0);
21400 match(Set dst (XorV src1 src2));
21401 format %{ "vpxor $dst,$src1,$src2\t! xor vectors" %}
21402 ins_encode %{
21403 int vlen_enc = vector_length_encoding(this);
21404 __ vpxor($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
21405 %}
21406 ins_pipe( pipe_slow );
21407 %}
21408
21409 instruct vxor_mem(vec dst, vec src, memory mem) %{
21410 predicate((UseAVX > 0) &&
21411 (Matcher::vector_length_in_bytes(n->in(1)) > 8));
21412 match(Set dst (XorV src (LoadVector mem)));
21413 format %{ "vpxor $dst,$src,$mem\t! xor vectors" %}
21414 ins_encode %{
21415 int vlen_enc = vector_length_encoding(this);
21416 __ vpxor($dst$$XMMRegister, $src$$XMMRegister, $mem$$Address, vlen_enc);
21417 %}
21418 ins_pipe( pipe_slow );
21419 %}
21420
21421 // --------------------------------- VectorCast --------------------------------------
21422
21423 instruct vcastBtoX(vec dst, vec src) %{
21424 predicate(VM_Version::supports_avx512vl() || Matcher::vector_element_basic_type(n) != T_DOUBLE);
21425 match(Set dst (VectorCastB2X src));
21426 format %{ "vector_cast_b2x $dst,$src\t!" %}
21427 ins_encode %{
21428 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21429 int vlen_enc = vector_length_encoding(this);
21430 __ vconvert_b2x(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21431 %}
21432 ins_pipe( pipe_slow );
21433 %}
21434
21435 instruct vcastBtoD(legVec dst, legVec src) %{
21436 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_element_basic_type(n) == T_DOUBLE);
21437 match(Set dst (VectorCastB2X src));
21438 format %{ "vector_cast_b2x $dst,$src\t!" %}
21439 ins_encode %{
21440 int vlen_enc = vector_length_encoding(this);
21441 __ vconvert_b2x(T_DOUBLE, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21442 %}
21443 ins_pipe( pipe_slow );
21444 %}
21445
21446 instruct castStoX(vec dst, vec src) %{
21447 predicate((UseAVX <= 2 || !VM_Version::supports_avx512vlbw()) &&
21448 Matcher::vector_length(n->in(1)) <= 8 && // src
21449 Matcher::vector_element_basic_type(n) == T_BYTE);
21450 match(Set dst (VectorCastS2X src));
21451 format %{ "vector_cast_s2x $dst,$src" %}
21452 ins_encode %{
21453 assert(UseAVX > 0, "required");
21454
21455 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), 0, noreg);
21456 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, 0);
21457 %}
21458 ins_pipe( pipe_slow );
21459 %}
21460
21461 instruct vcastStoX(vec dst, vec src, vec vtmp) %{
21462 predicate((UseAVX <= 2 || !VM_Version::supports_avx512vlbw()) &&
21463 Matcher::vector_length(n->in(1)) == 16 && // src
21464 Matcher::vector_element_basic_type(n) == T_BYTE);
21465 effect(TEMP dst, TEMP vtmp);
21466 match(Set dst (VectorCastS2X src));
21467 format %{ "vector_cast_s2x $dst,$src\t! using $vtmp as TEMP" %}
21468 ins_encode %{
21469 assert(UseAVX > 0, "required");
21470
21471 int vlen_enc = vector_length_encoding(Matcher::vector_length_in_bytes(this, $src));
21472 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_short_to_byte_mask()), vlen_enc, noreg);
21473 __ vextracti128($vtmp$$XMMRegister, $dst$$XMMRegister, 0x1);
21474 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0);
21475 %}
21476 ins_pipe( pipe_slow );
21477 %}
21478
21479 instruct vcastStoX_evex(vec dst, vec src) %{
21480 predicate((UseAVX > 2 && VM_Version::supports_avx512vlbw()) ||
21481 (Matcher::vector_length_in_bytes(n) >= Matcher::vector_length_in_bytes(n->in(1)))); // dst >= src
21482 match(Set dst (VectorCastS2X src));
21483 format %{ "vector_cast_s2x $dst,$src\t!" %}
21484 ins_encode %{
21485 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21486 int src_vlen_enc = vector_length_encoding(this, $src);
21487 int vlen_enc = vector_length_encoding(this);
21488 switch (to_elem_bt) {
21489 case T_BYTE:
21490 if (!VM_Version::supports_avx512vl()) {
21491 vlen_enc = Assembler::AVX_512bit;
21492 }
21493 __ evpmovwb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
21494 break;
21495 case T_INT:
21496 __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21497 break;
21498 case T_FLOAT:
21499 __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21500 __ vcvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21501 break;
21502 case T_LONG:
21503 __ vpmovsxwq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21504 break;
21505 case T_DOUBLE: {
21506 int mid_vlen_enc = (vlen_enc == Assembler::AVX_512bit) ? Assembler::AVX_256bit : Assembler::AVX_128bit;
21507 __ vpmovsxwd($dst$$XMMRegister, $src$$XMMRegister, mid_vlen_enc);
21508 __ vcvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21509 break;
21510 }
21511 default:
21512 ShouldNotReachHere();
21513 }
21514 %}
21515 ins_pipe( pipe_slow );
21516 %}
21517
21518 instruct castItoX(vec dst, vec src) %{
21519 predicate(UseAVX <= 2 &&
21520 (Matcher::vector_length_in_bytes(n->in(1)) <= 16) &&
21521 (Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)))); // dst < src
21522 match(Set dst (VectorCastI2X src));
21523 format %{ "vector_cast_i2x $dst,$src" %}
21524 ins_encode %{
21525 assert(UseAVX > 0, "required");
21526
21527 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21528 int vlen_enc = vector_length_encoding(this, $src);
21529
21530 if (to_elem_bt == T_BYTE) {
21531 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_byte_mask()), vlen_enc, noreg);
21532 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21533 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21534 } else {
21535 assert(to_elem_bt == T_SHORT, "%s", type2name(to_elem_bt));
21536 __ vpand($dst$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21537 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21538 }
21539 %}
21540 ins_pipe( pipe_slow );
21541 %}
21542
21543 instruct vcastItoX(vec dst, vec src, vec vtmp) %{
21544 predicate(UseAVX <= 2 &&
21545 (Matcher::vector_length_in_bytes(n->in(1)) == 32) &&
21546 (Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)))); // dst < src
21547 match(Set dst (VectorCastI2X src));
21548 format %{ "vector_cast_i2x $dst,$src\t! using $vtmp as TEMP" %}
21549 effect(TEMP dst, TEMP vtmp);
21550 ins_encode %{
21551 assert(UseAVX > 0, "required");
21552
21553 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21554 int vlen_enc = vector_length_encoding(this, $src);
21555
21556 if (to_elem_bt == T_BYTE) {
21557 __ vpand($vtmp$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_byte_mask()), vlen_enc, noreg);
21558 __ vextracti128($dst$$XMMRegister, $vtmp$$XMMRegister, 0x1);
21559 __ vpackusdw($dst$$XMMRegister, $vtmp$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21560 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21561 } else {
21562 assert(to_elem_bt == T_SHORT, "%s", type2name(to_elem_bt));
21563 __ vpand($vtmp$$XMMRegister, $src$$XMMRegister, ExternalAddress(vector_int_to_short_mask()), vlen_enc, noreg);
21564 __ vextracti128($dst$$XMMRegister, $vtmp$$XMMRegister, 0x1);
21565 __ vpackusdw($dst$$XMMRegister, $vtmp$$XMMRegister, $dst$$XMMRegister, vlen_enc);
21566 }
21567 %}
21568 ins_pipe( pipe_slow );
21569 %}
21570
21571 instruct vcastItoX_evex(vec dst, vec src) %{
21572 predicate(UseAVX > 2 ||
21573 (Matcher::vector_length_in_bytes(n) >= Matcher::vector_length_in_bytes(n->in(1)))); // dst >= src
21574 match(Set dst (VectorCastI2X src));
21575 format %{ "vector_cast_i2x $dst,$src\t!" %}
21576 ins_encode %{
21577 assert(UseAVX > 0, "required");
21578
21579 BasicType dst_elem_bt = Matcher::vector_element_basic_type(this);
21580 int src_vlen_enc = vector_length_encoding(this, $src);
21581 int dst_vlen_enc = vector_length_encoding(this);
21582 switch (dst_elem_bt) {
21583 case T_BYTE:
21584 if (!VM_Version::supports_avx512vl()) {
21585 src_vlen_enc = Assembler::AVX_512bit;
21586 }
21587 __ evpmovdb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
21588 break;
21589 case T_SHORT:
21590 if (!VM_Version::supports_avx512vl()) {
21591 src_vlen_enc = Assembler::AVX_512bit;
21592 }
21593 __ evpmovdw($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
21594 break;
21595 case T_FLOAT:
21596 __ vcvtdq2ps($dst$$XMMRegister, $src$$XMMRegister, dst_vlen_enc);
21597 break;
21598 case T_LONG:
21599 __ vpmovsxdq($dst$$XMMRegister, $src$$XMMRegister, dst_vlen_enc);
21600 break;
21601 case T_DOUBLE:
21602 __ vcvtdq2pd($dst$$XMMRegister, $src$$XMMRegister, dst_vlen_enc);
21603 break;
21604 default:
21605 ShouldNotReachHere();
21606 }
21607 %}
21608 ins_pipe( pipe_slow );
21609 %}
21610
21611 instruct vcastLtoBS(vec dst, vec src) %{
21612 predicate((Matcher::vector_element_basic_type(n) == T_BYTE || Matcher::vector_element_basic_type(n) == T_SHORT) &&
21613 UseAVX <= 2);
21614 match(Set dst (VectorCastL2X src));
21615 format %{ "vector_cast_l2x $dst,$src" %}
21616 ins_encode %{
21617 assert(UseAVX > 0, "required");
21618
21619 int vlen = Matcher::vector_length_in_bytes(this, $src);
21620 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21621 AddressLiteral mask_addr = (to_elem_bt == T_BYTE) ? ExternalAddress(vector_int_to_byte_mask())
21622 : ExternalAddress(vector_int_to_short_mask());
21623 if (vlen <= 16) {
21624 __ vpshufd($dst$$XMMRegister, $src$$XMMRegister, 8, Assembler::AVX_128bit);
21625 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, mask_addr, Assembler::AVX_128bit, noreg);
21626 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21627 } else {
21628 assert(vlen <= 32, "required");
21629 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 8, Assembler::AVX_256bit);
21630 __ vpermpd($dst$$XMMRegister, $dst$$XMMRegister, 8, Assembler::AVX_256bit);
21631 __ vpand($dst$$XMMRegister, $dst$$XMMRegister, mask_addr, Assembler::AVX_128bit, noreg);
21632 __ vpackusdw($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21633 }
21634 if (to_elem_bt == T_BYTE) {
21635 __ vpackuswb($dst$$XMMRegister, $dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_128bit);
21636 }
21637 %}
21638 ins_pipe( pipe_slow );
21639 %}
21640
21641 instruct vcastLtoX_evex(vec dst, vec src) %{
21642 predicate(UseAVX > 2 ||
21643 (Matcher::vector_element_basic_type(n) == T_INT ||
21644 Matcher::vector_element_basic_type(n) == T_FLOAT ||
21645 Matcher::vector_element_basic_type(n) == T_DOUBLE));
21646 match(Set dst (VectorCastL2X src));
21647 format %{ "vector_cast_l2x $dst,$src\t!" %}
21648 ins_encode %{
21649 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21650 int vlen = Matcher::vector_length_in_bytes(this, $src);
21651 int vlen_enc = vector_length_encoding(this, $src);
21652 switch (to_elem_bt) {
21653 case T_BYTE:
21654 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
21655 vlen_enc = Assembler::AVX_512bit;
21656 }
21657 __ evpmovqb($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21658 break;
21659 case T_SHORT:
21660 if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
21661 vlen_enc = Assembler::AVX_512bit;
21662 }
21663 __ evpmovqw($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21664 break;
21665 case T_INT:
21666 if (vlen == 8) {
21667 if ($dst$$XMMRegister != $src$$XMMRegister) {
21668 __ movflt($dst$$XMMRegister, $src$$XMMRegister);
21669 }
21670 } else if (vlen == 16) {
21671 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 8);
21672 } else if (vlen == 32) {
21673 if (UseAVX > 2) {
21674 if (!VM_Version::supports_avx512vl()) {
21675 vlen_enc = Assembler::AVX_512bit;
21676 }
21677 __ evpmovqd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21678 } else {
21679 __ vpermilps($dst$$XMMRegister, $src$$XMMRegister, 8, vlen_enc);
21680 __ vpermpd($dst$$XMMRegister, $dst$$XMMRegister, 8, vlen_enc);
21681 }
21682 } else { // vlen == 64
21683 __ evpmovqd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21684 }
21685 break;
21686 case T_FLOAT:
21687 assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "required");
21688 __ evcvtqq2ps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21689 break;
21690 case T_DOUBLE:
21691 assert(UseAVX > 2 && VM_Version::supports_avx512dq(), "required");
21692 __ evcvtqq2pd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21693 break;
21694
21695 default: assert(false, "%s", type2name(to_elem_bt));
21696 }
21697 %}
21698 ins_pipe( pipe_slow );
21699 %}
21700
21701 instruct vcastFtoD_reg(vec dst, vec src) %{
21702 predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
21703 match(Set dst (VectorCastF2X src));
21704 format %{ "vector_cast_f2d $dst,$src\t!" %}
21705 ins_encode %{
21706 int vlen_enc = vector_length_encoding(this);
21707 __ vcvtps2pd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21708 %}
21709 ins_pipe( pipe_slow );
21710 %}
21711
21712
21713 instruct castFtoX_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, rFlagsReg cr) %{
21714 predicate(!VM_Version::supports_avx10_2() &&
21715 !VM_Version::supports_avx512vl() &&
21716 Matcher::vector_length_in_bytes(n->in(1)) < 64 &&
21717 type2aelembytes(Matcher::vector_element_basic_type(n)) <= 4 &&
21718 is_integral_type(Matcher::vector_element_basic_type(n)));
21719 match(Set dst (VectorCastF2X src));
21720 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, KILL cr);
21721 format %{ "vector_cast_f2x $dst,$src\t! using $xtmp1, $xtmp2, $xtmp3 and $xtmp4 as TEMP" %}
21722 ins_encode %{
21723 int vlen_enc = vector_length_encoding(this, $src);
21724 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21725 // JDK-8292878 removed the need for an explicit scratch register needed to load greater than
21726 // 32 bit addresses for register indirect addressing mode since stub constants
21727 // are part of code cache and there is a cap of 2G on ReservedCodeCacheSize currently.
21728 // However, targets are free to increase this limit, but having a large code cache size
21729 // greater than 2G looks unreasonable in practical scenario, on the hind side with given
21730 // cap we save a temporary register allocation which in limiting case can prevent
21731 // spilling in high register pressure blocks.
21732 __ vector_castF2X_avx(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
21733 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister,
21734 ExternalAddress(vector_float_signflip()), noreg, vlen_enc);
21735 %}
21736 ins_pipe( pipe_slow );
21737 %}
21738
21739 instruct castFtoX_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
21740 predicate(!VM_Version::supports_avx10_2() &&
21741 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n->in(1)) == 64) &&
21742 is_integral_type(Matcher::vector_element_basic_type(n)));
21743 match(Set dst (VectorCastF2X src));
21744 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
21745 format %{ "vector_cast_f2x $dst,$src\t! using $xtmp1, $xtmp2, $ktmp1 and $ktmp2 as TEMP" %}
21746 ins_encode %{
21747 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21748 if (to_elem_bt == T_LONG) {
21749 int vlen_enc = vector_length_encoding(this);
21750 __ vector_castF2L_evex($dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
21751 $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister,
21752 ExternalAddress(vector_double_signflip()), noreg, vlen_enc);
21753 } else {
21754 int vlen_enc = vector_length_encoding(this, $src);
21755 __ vector_castF2X_evex(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
21756 $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister,
21757 ExternalAddress(vector_float_signflip()), noreg, vlen_enc);
21758 }
21759 %}
21760 ins_pipe( pipe_slow );
21761 %}
21762
21763 instruct castFtoX_reg_avx10_2(vec dst, vec src) %{
21764 predicate(VM_Version::supports_avx10_2() &&
21765 is_integral_type(Matcher::vector_element_basic_type(n)));
21766 match(Set dst (VectorCastF2X src));
21767 format %{ "vector_cast_f2x_avx10_2 $dst, $src\t!" %}
21768 ins_encode %{
21769 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21770 int vlen_enc = (to_elem_bt == T_LONG) ? vector_length_encoding(this) : vector_length_encoding(this, $src);
21771 __ vector_castF2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21772 %}
21773 ins_pipe( pipe_slow );
21774 %}
21775
21776 instruct castFtoX_mem_avx10_2(vec dst, memory src) %{
21777 predicate(VM_Version::supports_avx10_2() &&
21778 is_integral_type(Matcher::vector_element_basic_type(n)));
21779 match(Set dst (VectorCastF2X (LoadVector src)));
21780 format %{ "vector_cast_f2x_avx10_2 $dst, $src\t!" %}
21781 ins_encode %{
21782 int vlen = Matcher::vector_length(this);
21783 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21784 int vlen_enc = (to_elem_bt == T_LONG) ? vector_length_encoding(this) : vector_length_encoding(vlen * sizeof(jfloat));
21785 __ vector_castF2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$Address, vlen_enc);
21786 %}
21787 ins_pipe( pipe_slow );
21788 %}
21789
21790 instruct vcastDtoF_reg(vec dst, vec src) %{
21791 predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
21792 match(Set dst (VectorCastD2X src));
21793 format %{ "vector_cast_d2x $dst,$src\t!" %}
21794 ins_encode %{
21795 int vlen_enc = vector_length_encoding(this, $src);
21796 __ vcvtpd2ps($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21797 %}
21798 ins_pipe( pipe_slow );
21799 %}
21800
21801 instruct castDtoX_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, vec xtmp5, rFlagsReg cr) %{
21802 predicate(!VM_Version::supports_avx10_2() &&
21803 !VM_Version::supports_avx512vl() &&
21804 Matcher::vector_length_in_bytes(n->in(1)) < 64 &&
21805 is_integral_type(Matcher::vector_element_basic_type(n)));
21806 match(Set dst (VectorCastD2X src));
21807 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, TEMP xtmp5, KILL cr);
21808 format %{ "vector_cast_d2x $dst,$src\t! using $xtmp1, $xtmp2, $xtmp3, $xtmp4 and $xtmp5 as TEMP" %}
21809 ins_encode %{
21810 int vlen_enc = vector_length_encoding(this, $src);
21811 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21812 __ vector_castD2X_avx(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
21813 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister, $xtmp5$$XMMRegister,
21814 ExternalAddress(vector_float_signflip()), noreg, vlen_enc);
21815 %}
21816 ins_pipe( pipe_slow );
21817 %}
21818
21819 instruct castDtoX_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
21820 predicate(!VM_Version::supports_avx10_2() &&
21821 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n->in(1)) == 64) &&
21822 is_integral_type(Matcher::vector_element_basic_type(n)));
21823 match(Set dst (VectorCastD2X src));
21824 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
21825 format %{ "vector_cast_d2x $dst,$src\t! using $xtmp1, $xtmp2, $ktmp1 and $ktmp2 as TEMP" %}
21826 ins_encode %{
21827 int vlen_enc = vector_length_encoding(this, $src);
21828 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21829 AddressLiteral signflip = VM_Version::supports_avx512dq() ? ExternalAddress(vector_double_signflip()) :
21830 ExternalAddress(vector_float_signflip());
21831 __ vector_castD2X_evex(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
21832 $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister, signflip, noreg, vlen_enc);
21833 %}
21834 ins_pipe( pipe_slow );
21835 %}
21836
21837 instruct castDtoX_reg_avx10_2(vec dst, vec src) %{
21838 predicate(VM_Version::supports_avx10_2() &&
21839 is_integral_type(Matcher::vector_element_basic_type(n)));
21840 match(Set dst (VectorCastD2X src));
21841 format %{ "vector_cast_d2x_avx10_2 $dst, $src\t!" %}
21842 ins_encode %{
21843 int vlen_enc = vector_length_encoding(this, $src);
21844 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21845 __ vector_castD2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
21846 %}
21847 ins_pipe( pipe_slow );
21848 %}
21849
21850 instruct castDtoX_mem_avx10_2(vec dst, memory src) %{
21851 predicate(VM_Version::supports_avx10_2() &&
21852 is_integral_type(Matcher::vector_element_basic_type(n)));
21853 match(Set dst (VectorCastD2X (LoadVector src)));
21854 format %{ "vector_cast_d2x_avx10_2 $dst, $src\t!" %}
21855 ins_encode %{
21856 int vlen = Matcher::vector_length(this);
21857 int vlen_enc = vector_length_encoding(vlen * sizeof(jdouble));
21858 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21859 __ vector_castD2X_avx10_2(to_elem_bt, $dst$$XMMRegister, $src$$Address, vlen_enc);
21860 %}
21861 ins_pipe( pipe_slow );
21862 %}
21863
21864 instruct vucast(vec dst, vec src) %{
21865 match(Set dst (VectorUCastB2X src));
21866 match(Set dst (VectorUCastS2X src));
21867 match(Set dst (VectorUCastI2X src));
21868 format %{ "vector_ucast $dst,$src\t!" %}
21869 ins_encode %{
21870 assert(UseAVX > 0, "required");
21871
21872 BasicType from_elem_bt = Matcher::vector_element_basic_type(this, $src);
21873 BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
21874 int vlen_enc = vector_length_encoding(this);
21875 __ vector_unsigned_cast($dst$$XMMRegister, $src$$XMMRegister, vlen_enc, from_elem_bt, to_elem_bt);
21876 %}
21877 ins_pipe( pipe_slow );
21878 %}
21879
21880 instruct vround_float_avx(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, rFlagsReg cr) %{
21881 predicate(!VM_Version::supports_avx512vl() &&
21882 Matcher::vector_length_in_bytes(n) < 64 &&
21883 Matcher::vector_element_basic_type(n) == T_INT);
21884 match(Set dst (RoundVF src));
21885 effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, KILL cr);
21886 format %{ "vector_round_float $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $xtmp3, $xtmp4 as TEMP" %}
21887 ins_encode %{
21888 int vlen_enc = vector_length_encoding(this);
21889 InternalAddress new_mxcsr = $constantaddress((jint)(EnableX86ECoreOpts ? 0x3FBF : 0x3F80));
21890 __ vector_round_float_avx($dst$$XMMRegister, $src$$XMMRegister,
21891 ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), new_mxcsr, vlen_enc,
21892 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister);
21893 %}
21894 ins_pipe( pipe_slow );
21895 %}
21896
21897 instruct vround_float_evex(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
21898 predicate((VM_Version::supports_avx512vl() ||
21899 Matcher::vector_length_in_bytes(n) == 64) &&
21900 Matcher::vector_element_basic_type(n) == T_INT);
21901 match(Set dst (RoundVF src));
21902 effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
21903 format %{ "vector_round_float $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $ktmp1, $ktmp2 as TEMP" %}
21904 ins_encode %{
21905 int vlen_enc = vector_length_encoding(this);
21906 InternalAddress new_mxcsr = $constantaddress((jint)(EnableX86ECoreOpts ? 0x3FBF : 0x3F80));
21907 __ vector_round_float_evex($dst$$XMMRegister, $src$$XMMRegister,
21908 ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), new_mxcsr, vlen_enc,
21909 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
21910 %}
21911 ins_pipe( pipe_slow );
21912 %}
21913
21914 instruct vround_reg_evex(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
21915 predicate(Matcher::vector_element_basic_type(n) == T_LONG);
21916 match(Set dst (RoundVD src));
21917 effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
21918 format %{ "vector_round_long $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $ktmp1, $ktmp2 as TEMP" %}
21919 ins_encode %{
21920 int vlen_enc = vector_length_encoding(this);
21921 InternalAddress new_mxcsr = $constantaddress((jint)(EnableX86ECoreOpts ? 0x3FBF : 0x3F80));
21922 __ vector_round_double_evex($dst$$XMMRegister, $src$$XMMRegister,
21923 ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), new_mxcsr, vlen_enc,
21924 $tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
21925 %}
21926 ins_pipe( pipe_slow );
21927 %}
21928
21929 // --------------------------------- VectorMaskCmp --------------------------------------
21930
21931 instruct vcmpFD(legVec dst, legVec src1, legVec src2, immI8 cond) %{
21932 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
21933 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 8 && // src1
21934 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
21935 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
21936 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
21937 format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
21938 ins_encode %{
21939 int vlen_enc = vector_length_encoding(this, $src1);
21940 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
21941 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
21942 __ vcmpps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
21943 } else {
21944 __ vcmppd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
21945 }
21946 %}
21947 ins_pipe( pipe_slow );
21948 %}
21949
21950 instruct evcmpFD64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
21951 predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64 && // src1
21952 n->bottom_type()->isa_pvectmask() == nullptr &&
21953 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
21954 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
21955 effect(TEMP ktmp);
21956 format %{ "vector_compare $dst,$src1,$src2,$cond" %}
21957 ins_encode %{
21958 int vlen_enc = Assembler::AVX_512bit;
21959 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
21960 KRegister mask = k0; // The comparison itself is not being masked.
21961 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
21962 __ evcmpps($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
21963 __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, noreg);
21964 } else {
21965 __ evcmppd($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
21966 __ evmovdquq($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), false, vlen_enc, noreg);
21967 }
21968 %}
21969 ins_pipe( pipe_slow );
21970 %}
21971
21972 instruct evcmpFD(kReg dst, vec src1, vec src2, immI8 cond) %{
21973 predicate(n->bottom_type()->isa_pvectmask() &&
21974 is_floating_point_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1 T_FLOAT, T_DOUBLE
21975 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
21976 format %{ "vector_compare_evex $dst,$src1,$src2,$cond\t!" %}
21977 ins_encode %{
21978 assert(bottom_type()->isa_pvectmask(), "TypePVectMask expected");
21979 int vlen_enc = vector_length_encoding(this, $src1);
21980 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
21981 KRegister mask = k0; // The comparison itself is not being masked.
21982 if (Matcher::vector_element_basic_type(this, $src1) == T_FLOAT) {
21983 __ evcmpps($dst$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
21984 } else {
21985 __ evcmppd($dst$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
21986 }
21987 %}
21988 ins_pipe( pipe_slow );
21989 %}
21990
21991 instruct vcmp_direct(legVec dst, legVec src1, legVec src2, immI8 cond) %{
21992 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
21993 !Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
21994 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
21995 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
21996 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1))) &&
21997 (n->in(2)->get_int() == BoolTest::eq ||
21998 n->in(2)->get_int() == BoolTest::lt ||
21999 n->in(2)->get_int() == BoolTest::gt)); // cond
22000 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22001 format %{ "vector_compare $dst,$src1,$src2,$cond\t!" %}
22002 ins_encode %{
22003 int vlen_enc = vector_length_encoding(this, $src1);
22004 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22005 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
22006 __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, cmp, ww, vlen_enc);
22007 %}
22008 ins_pipe( pipe_slow );
22009 %}
22010
22011 instruct vcmp_negate(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
22012 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
22013 !Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
22014 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
22015 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
22016 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1))) &&
22017 (n->in(2)->get_int() == BoolTest::ne ||
22018 n->in(2)->get_int() == BoolTest::le ||
22019 n->in(2)->get_int() == BoolTest::ge)); // cond
22020 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22021 effect(TEMP dst, TEMP xtmp);
22022 format %{ "vector_compare $dst,$src1,$src2,$cond\t! using $xtmp as TEMP" %}
22023 ins_encode %{
22024 int vlen_enc = vector_length_encoding(this, $src1);
22025 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22026 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
22027 __ vpcmpCCW($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $xtmp$$XMMRegister, cmp, ww, vlen_enc);
22028 %}
22029 ins_pipe( pipe_slow );
22030 %}
22031
22032 instruct vcmpu(legVec dst, legVec src1, legVec src2, immI8 cond, legVec xtmp) %{
22033 predicate(n->bottom_type()->isa_pvectmask() == nullptr &&
22034 Matcher::is_unsigned_booltest_pred(n->in(2)->get_int()) &&
22035 Matcher::vector_length_in_bytes(n->in(1)->in(1)) >= 4 && // src1
22036 Matcher::vector_length_in_bytes(n->in(1)->in(1)) <= 32 && // src1
22037 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
22038 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22039 effect(TEMP dst, TEMP xtmp);
22040 format %{ "vector_compareu $dst,$src1,$src2,$cond\t! using $xtmp as TEMP" %}
22041 ins_encode %{
22042 InternalAddress flip_bit = $constantaddress(high_bit_set(Matcher::vector_element_basic_type(this, $src1)));
22043 int vlen_enc = vector_length_encoding(this, $src1);
22044 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22045 Assembler::Width ww = widthForType(Matcher::vector_element_basic_type(this, $src1));
22046
22047 if (vlen_enc == Assembler::AVX_128bit) {
22048 __ vmovddup($xtmp$$XMMRegister, flip_bit, vlen_enc, noreg);
22049 } else {
22050 __ vbroadcastsd($xtmp$$XMMRegister, flip_bit, vlen_enc, noreg);
22051 }
22052 __ vpxor($dst$$XMMRegister, $xtmp$$XMMRegister, $src1$$XMMRegister, vlen_enc);
22053 __ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $src2$$XMMRegister, vlen_enc);
22054 __ vpcmpCCW($dst$$XMMRegister, $dst$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, cmp, ww, vlen_enc);
22055 %}
22056 ins_pipe( pipe_slow );
22057 %}
22058
22059 instruct vcmp64(vec dst, vec src1, vec src2, immI8 cond, kReg ktmp) %{
22060 predicate((n->bottom_type()->isa_pvectmask() == nullptr &&
22061 Matcher::vector_length_in_bytes(n->in(1)->in(1)) == 64) && // src1
22062 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
22063 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22064 effect(TEMP ktmp);
22065 format %{ "vector_compare $dst,$src1,$src2,$cond" %}
22066 ins_encode %{
22067 assert(UseAVX > 2, "required");
22068
22069 int vlen_enc = vector_length_encoding(this, $src1);
22070 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22071 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
22072 KRegister mask = k0; // The comparison itself is not being masked.
22073 bool merge = false;
22074 BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
22075
22076 switch (src1_elem_bt) {
22077 case T_INT: {
22078 __ evpcmpd($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22079 __ evmovdqul($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), merge, vlen_enc, noreg);
22080 break;
22081 }
22082 case T_LONG: {
22083 __ evpcmpq($ktmp$$KRegister, mask, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22084 __ evmovdquq($dst$$XMMRegister, $ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), merge, vlen_enc, noreg);
22085 break;
22086 }
22087 default: assert(false, "%s", type2name(src1_elem_bt));
22088 }
22089 %}
22090 ins_pipe( pipe_slow );
22091 %}
22092
22093
22094 instruct evcmp(kReg dst, vec src1, vec src2, immI8 cond) %{
22095 predicate(n->bottom_type()->isa_pvectmask() &&
22096 is_integral_type(Matcher::vector_element_basic_type(n->in(1)->in(1)))); // src1
22097 match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
22098 format %{ "vector_compared_evex $dst,$src1,$src2,$cond\t!" %}
22099 ins_encode %{
22100 assert(UseAVX > 2, "required");
22101 assert(bottom_type()->isa_pvectmask(), "TypePVectMask expected");
22102
22103 int vlen_enc = vector_length_encoding(this, $src1);
22104 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
22105 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
22106 BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
22107
22108 // Comparison i
22109 switch (src1_elem_bt) {
22110 case T_BYTE: {
22111 __ evpcmpb($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22112 break;
22113 }
22114 case T_SHORT: {
22115 __ evpcmpw($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22116 break;
22117 }
22118 case T_INT: {
22119 __ evpcmpd($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22120 break;
22121 }
22122 case T_LONG: {
22123 __ evpcmpq($dst$$KRegister, k0, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
22124 break;
22125 }
22126 default: assert(false, "%s", type2name(src1_elem_bt));
22127 }
22128 %}
22129 ins_pipe( pipe_slow );
22130 %}
22131
22132 // Extract
22133
22134 instruct extractI(rRegI dst, legVec src, immU8 idx) %{
22135 predicate(Matcher::vector_length_in_bytes(n->in(1)) <= 16); // src
22136 match(Set dst (ExtractI src idx));
22137 match(Set dst (ExtractS src idx));
22138 match(Set dst (ExtractB src idx));
22139 format %{ "extractI $dst,$src,$idx\t!" %}
22140 ins_encode %{
22141 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22142
22143 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
22144 __ get_elem(elem_bt, $dst$$Register, $src$$XMMRegister, $idx$$constant);
22145 %}
22146 ins_pipe( pipe_slow );
22147 %}
22148
22149 instruct vextractI(rRegI dst, legVec src, immI idx, legVec vtmp) %{
22150 predicate(Matcher::vector_length_in_bytes(n->in(1)) == 32 || // src
22151 Matcher::vector_length_in_bytes(n->in(1)) == 64); // src
22152 match(Set dst (ExtractI src idx));
22153 match(Set dst (ExtractS src idx));
22154 match(Set dst (ExtractB src idx));
22155 effect(TEMP vtmp);
22156 format %{ "vextractI $dst,$src,$idx\t! using $vtmp as TEMP" %}
22157 ins_encode %{
22158 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22159
22160 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src);
22161 XMMRegister lane_xmm = __ get_lane(elem_bt, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22162 __ get_elem(elem_bt, $dst$$Register, lane_xmm, $idx$$constant);
22163 %}
22164 ins_pipe( pipe_slow );
22165 %}
22166
22167 instruct extractL(rRegL dst, legVec src, immU8 idx) %{
22168 predicate(Matcher::vector_length(n->in(1)) <= 2); // src
22169 match(Set dst (ExtractL src idx));
22170 format %{ "extractL $dst,$src,$idx\t!" %}
22171 ins_encode %{
22172 assert(UseSSE >= 4, "required");
22173 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22174
22175 __ get_elem(T_LONG, $dst$$Register, $src$$XMMRegister, $idx$$constant);
22176 %}
22177 ins_pipe( pipe_slow );
22178 %}
22179
22180 instruct vextractL(rRegL dst, legVec src, immU8 idx, legVec vtmp) %{
22181 predicate(Matcher::vector_length(n->in(1)) == 4 || // src
22182 Matcher::vector_length(n->in(1)) == 8); // src
22183 match(Set dst (ExtractL src idx));
22184 effect(TEMP vtmp);
22185 format %{ "vextractL $dst,$src,$idx\t! using $vtmp as TEMP" %}
22186 ins_encode %{
22187 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22188
22189 XMMRegister lane_reg = __ get_lane(T_LONG, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22190 __ get_elem(T_LONG, $dst$$Register, lane_reg, $idx$$constant);
22191 %}
22192 ins_pipe( pipe_slow );
22193 %}
22194
22195 instruct extractF(legRegF dst, legVec src, immU8 idx, legVec vtmp) %{
22196 predicate(Matcher::vector_length(n->in(1)) <= 4);
22197 match(Set dst (ExtractF src idx));
22198 effect(TEMP dst, TEMP vtmp);
22199 format %{ "extractF $dst,$src,$idx\t! using $vtmp as TEMP" %}
22200 ins_encode %{
22201 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22202
22203 __ get_elem(T_FLOAT, $dst$$XMMRegister, $src$$XMMRegister, $idx$$constant, $vtmp$$XMMRegister);
22204 %}
22205 ins_pipe( pipe_slow );
22206 %}
22207
22208 instruct vextractF(legRegF dst, legVec src, immU8 idx, legVec vtmp) %{
22209 predicate(Matcher::vector_length(n->in(1)/*src*/) == 8 ||
22210 Matcher::vector_length(n->in(1)/*src*/) == 16);
22211 match(Set dst (ExtractF src idx));
22212 effect(TEMP vtmp);
22213 format %{ "vextractF $dst,$src,$idx\t! using $vtmp as TEMP" %}
22214 ins_encode %{
22215 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22216
22217 XMMRegister lane_reg = __ get_lane(T_FLOAT, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22218 __ get_elem(T_FLOAT, $dst$$XMMRegister, lane_reg, $idx$$constant);
22219 %}
22220 ins_pipe( pipe_slow );
22221 %}
22222
22223 instruct extractD(legRegD dst, legVec src, immU8 idx) %{
22224 predicate(Matcher::vector_length(n->in(1)) == 2); // src
22225 match(Set dst (ExtractD src idx));
22226 format %{ "extractD $dst,$src,$idx\t!" %}
22227 ins_encode %{
22228 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22229
22230 __ get_elem(T_DOUBLE, $dst$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22231 %}
22232 ins_pipe( pipe_slow );
22233 %}
22234
22235 instruct vextractD(legRegD dst, legVec src, immU8 idx, legVec vtmp) %{
22236 predicate(Matcher::vector_length(n->in(1)) == 4 || // src
22237 Matcher::vector_length(n->in(1)) == 8); // src
22238 match(Set dst (ExtractD src idx));
22239 effect(TEMP vtmp);
22240 format %{ "vextractD $dst,$src,$idx\t! using $vtmp as TEMP" %}
22241 ins_encode %{
22242 assert($idx$$constant < (int)Matcher::vector_length(this, $src), "out of bounds");
22243
22244 XMMRegister lane_reg = __ get_lane(T_DOUBLE, $vtmp$$XMMRegister, $src$$XMMRegister, $idx$$constant);
22245 __ get_elem(T_DOUBLE, $dst$$XMMRegister, lane_reg, $idx$$constant);
22246 %}
22247 ins_pipe( pipe_slow );
22248 %}
22249
22250 // --------------------------------- Vector Blend --------------------------------------
22251
22252 instruct blendvp(vec dst, vec src, vec mask, rxmm0 tmp) %{
22253 predicate(UseAVX == 0);
22254 match(Set dst (VectorBlend (Binary dst src) mask));
22255 format %{ "vector_blend $dst,$src,$mask\t! using $tmp as TEMP" %}
22256 effect(TEMP tmp);
22257 ins_encode %{
22258 assert(UseSSE >= 4, "required");
22259
22260 if ($mask$$XMMRegister != $tmp$$XMMRegister) {
22261 __ movdqu($tmp$$XMMRegister, $mask$$XMMRegister);
22262 }
22263 __ pblendvb($dst$$XMMRegister, $src$$XMMRegister); // uses xmm0 as mask
22264 %}
22265 ins_pipe( pipe_slow );
22266 %}
22267
22268 instruct vblendvpI(legVec dst, legVec src1, legVec src2, legVec mask) %{
22269 predicate(UseAVX > 0 && !EnableX86ECoreOpts &&
22270 n->in(2)->bottom_type()->isa_pvectmask() == nullptr &&
22271 Matcher::vector_length_in_bytes(n) <= 32 &&
22272 is_integral_type(Matcher::vector_element_basic_type(n)));
22273 match(Set dst (VectorBlend (Binary src1 src2) mask));
22274 format %{ "vector_blend $dst,$src1,$src2,$mask\t!" %}
22275 ins_encode %{
22276 int vlen_enc = vector_length_encoding(this);
22277 __ vpblendvb($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $mask$$XMMRegister, vlen_enc);
22278 %}
22279 ins_pipe( pipe_slow );
22280 %}
22281
22282 instruct vblendvpFD(legVec dst, legVec src1, legVec src2, legVec mask) %{
22283 predicate(UseAVX > 0 && !EnableX86ECoreOpts &&
22284 n->in(2)->bottom_type()->isa_pvectmask() == nullptr &&
22285 Matcher::vector_length_in_bytes(n) <= 32 &&
22286 !is_integral_type(Matcher::vector_element_basic_type(n)));
22287 match(Set dst (VectorBlend (Binary src1 src2) mask));
22288 format %{ "vector_blend $dst,$src1,$src2,$mask\t!" %}
22289 ins_encode %{
22290 int vlen_enc = vector_length_encoding(this);
22291 __ vblendvps($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $mask$$XMMRegister, vlen_enc);
22292 %}
22293 ins_pipe( pipe_slow );
22294 %}
22295
22296 instruct vblendvp(legVec dst, legVec src1, legVec src2, legVec mask, legVec vtmp) %{
22297 predicate(UseAVX > 0 && EnableX86ECoreOpts &&
22298 n->in(2)->bottom_type()->isa_pvectmask() == nullptr &&
22299 Matcher::vector_length_in_bytes(n) <= 32);
22300 match(Set dst (VectorBlend (Binary src1 src2) mask));
22301 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using $vtmp as TEMP" %}
22302 effect(TEMP vtmp, TEMP dst);
22303 ins_encode %{
22304 int vlen_enc = vector_length_encoding(this);
22305 __ vpandn($vtmp$$XMMRegister, $mask$$XMMRegister, $src1$$XMMRegister, vlen_enc);
22306 __ vpand ($dst$$XMMRegister, $mask$$XMMRegister, $src2$$XMMRegister, vlen_enc);
22307 __ vpor ($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22308 %}
22309 ins_pipe( pipe_slow );
22310 %}
22311
22312 instruct evblendvp64(vec dst, vec src1, vec src2, vec mask, kReg ktmp) %{
22313 predicate(Matcher::vector_length_in_bytes(n) == 64 &&
22314 n->in(2)->bottom_type()->isa_pvectmask() == nullptr);
22315 match(Set dst (VectorBlend (Binary src1 src2) mask));
22316 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
22317 effect(TEMP ktmp);
22318 ins_encode %{
22319 int vlen_enc = Assembler::AVX_512bit;
22320 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22321 __ evpcmp(elem_bt, $ktmp$$KRegister, k0, $mask$$XMMRegister, ExternalAddress(vector_all_bits_set()), Assembler::eq, vlen_enc, noreg);
22322 __ evpblend(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
22323 %}
22324 ins_pipe( pipe_slow );
22325 %}
22326
22327
22328 instruct evblendvp64_masked(vec dst, vec src1, vec src2, kReg mask) %{
22329 predicate(n->in(2)->bottom_type()->isa_pvectmask() &&
22330 (!is_subword_type(Matcher::vector_element_basic_type(n)) ||
22331 VM_Version::supports_avx512bw()));
22332 match(Set dst (VectorBlend (Binary src1 src2) mask));
22333 format %{ "vector_blend $dst,$src1,$src2,$mask\t! using k2 as TEMP" %}
22334 ins_encode %{
22335 int vlen_enc = vector_length_encoding(this);
22336 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22337 __ evpblend(elem_bt, $dst$$XMMRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
22338 %}
22339 ins_pipe( pipe_slow );
22340 %}
22341
22342 // --------------------------------- ABS --------------------------------------
22343 // a = |a|
22344 instruct vabsB_reg(vec dst, vec src) %{
22345 match(Set dst (AbsVB src));
22346 format %{ "vabsb $dst,$src\t# $dst = |$src| abs packedB" %}
22347 ins_encode %{
22348 uint vlen = Matcher::vector_length(this);
22349 if (vlen <= 16) {
22350 __ pabsb($dst$$XMMRegister, $src$$XMMRegister);
22351 } else {
22352 int vlen_enc = vector_length_encoding(this);
22353 __ vpabsb($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22354 }
22355 %}
22356 ins_pipe( pipe_slow );
22357 %}
22358
22359 instruct vabsS_reg(vec dst, vec src) %{
22360 match(Set dst (AbsVS src));
22361 format %{ "vabsw $dst,$src\t# $dst = |$src| abs packedS" %}
22362 ins_encode %{
22363 uint vlen = Matcher::vector_length(this);
22364 if (vlen <= 8) {
22365 __ pabsw($dst$$XMMRegister, $src$$XMMRegister);
22366 } else {
22367 int vlen_enc = vector_length_encoding(this);
22368 __ vpabsw($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22369 }
22370 %}
22371 ins_pipe( pipe_slow );
22372 %}
22373
22374 instruct vabsI_reg(vec dst, vec src) %{
22375 match(Set dst (AbsVI src));
22376 format %{ "pabsd $dst,$src\t# $dst = |$src| abs packedI" %}
22377 ins_encode %{
22378 uint vlen = Matcher::vector_length(this);
22379 if (vlen <= 4) {
22380 __ pabsd($dst$$XMMRegister, $src$$XMMRegister);
22381 } else {
22382 int vlen_enc = vector_length_encoding(this);
22383 __ vpabsd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22384 }
22385 %}
22386 ins_pipe( pipe_slow );
22387 %}
22388
22389 instruct vabsL_reg(vec dst, vec src) %{
22390 match(Set dst (AbsVL src));
22391 format %{ "evpabsq $dst,$src\t# $dst = |$src| abs packedL" %}
22392 ins_encode %{
22393 assert(UseAVX > 2, "required");
22394 int vlen_enc = vector_length_encoding(this);
22395 if (!VM_Version::supports_avx512vl()) {
22396 vlen_enc = Assembler::AVX_512bit;
22397 }
22398 __ evpabsq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22399 %}
22400 ins_pipe( pipe_slow );
22401 %}
22402
22403 // --------------------------------- ABSNEG --------------------------------------
22404
22405 instruct vabsnegF(vec dst, vec src) %{
22406 predicate(Matcher::vector_length(n) != 4); // handled by 1-operand instruction vabsneg4F
22407 match(Set dst (AbsVF src));
22408 match(Set dst (NegVF src));
22409 format %{ "vabsnegf $dst,$src,[mask]\t# absneg packedF" %}
22410 ins_cost(150);
22411 ins_encode %{
22412 int opcode = this->ideal_Opcode();
22413 int vlen = Matcher::vector_length(this);
22414 if (vlen == 2) {
22415 __ vabsnegf(opcode, $dst$$XMMRegister, $src$$XMMRegister);
22416 } else {
22417 assert(vlen == 8 || vlen == 16, "required");
22418 int vlen_enc = vector_length_encoding(this);
22419 __ vabsnegf(opcode, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22420 }
22421 %}
22422 ins_pipe( pipe_slow );
22423 %}
22424
22425 instruct vabsneg4F(vec dst) %{
22426 predicate(Matcher::vector_length(n) == 4);
22427 match(Set dst (AbsVF dst));
22428 match(Set dst (NegVF dst));
22429 format %{ "vabsnegf $dst,[mask]\t# absneg packed4F" %}
22430 ins_cost(150);
22431 ins_encode %{
22432 int opcode = this->ideal_Opcode();
22433 __ vabsnegf(opcode, $dst$$XMMRegister, $dst$$XMMRegister);
22434 %}
22435 ins_pipe( pipe_slow );
22436 %}
22437
22438 instruct vabsnegD(vec dst, vec src) %{
22439 match(Set dst (AbsVD src));
22440 match(Set dst (NegVD src));
22441 format %{ "vabsnegd $dst,$src,[mask]\t# absneg packedD" %}
22442 ins_encode %{
22443 int opcode = this->ideal_Opcode();
22444 uint vlen = Matcher::vector_length(this);
22445 if (vlen == 2) {
22446 __ vabsnegd(opcode, $dst$$XMMRegister, $src$$XMMRegister);
22447 } else {
22448 int vlen_enc = vector_length_encoding(this);
22449 __ vabsnegd(opcode, $dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
22450 }
22451 %}
22452 ins_pipe( pipe_slow );
22453 %}
22454
22455 //------------------------------------- VectorTest --------------------------------------------
22456
22457 instruct vptest_lt16(rFlagsRegU cr, legVec src1, legVec src2, legVec vtmp) %{
22458 predicate(Matcher::vector_length_in_bytes(n->in(1)) < 16);
22459 match(Set cr (VectorTest src1 src2));
22460 effect(TEMP vtmp);
22461 format %{ "vptest_lt16 $src1, $src2\t! using $vtmp as TEMP" %}
22462 ins_encode %{
22463 BasicType bt = Matcher::vector_element_basic_type(this, $src1);
22464 int vlen = Matcher::vector_length_in_bytes(this, $src1);
22465 __ vectortest(bt, $src1$$XMMRegister, $src2$$XMMRegister, $vtmp$$XMMRegister, vlen);
22466 %}
22467 ins_pipe( pipe_slow );
22468 %}
22469
22470 instruct vptest_ge16(rFlagsRegU cr, legVec src1, legVec src2) %{
22471 predicate(Matcher::vector_length_in_bytes(n->in(1)) >= 16);
22472 match(Set cr (VectorTest src1 src2));
22473 format %{ "vptest_ge16 $src1, $src2\n\t" %}
22474 ins_encode %{
22475 BasicType bt = Matcher::vector_element_basic_type(this, $src1);
22476 int vlen = Matcher::vector_length_in_bytes(this, $src1);
22477 __ vectortest(bt, $src1$$XMMRegister, $src2$$XMMRegister, xnoreg, vlen);
22478 %}
22479 ins_pipe( pipe_slow );
22480 %}
22481
22482 instruct ktest_alltrue_le8(rFlagsRegU cr, kReg src1, kReg src2, rRegI tmp) %{
22483 predicate((Matcher::vector_length(n->in(1)) < 8 ||
22484 (Matcher::vector_length(n->in(1)) == 8 && !VM_Version::supports_avx512dq())) &&
22485 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
22486 match(Set cr (VectorTest src1 src2));
22487 effect(TEMP tmp);
22488 format %{ "ktest_alltrue_le8 $src1, $src2\t! using $tmp as TEMP" %}
22489 ins_encode %{
22490 uint masklen = Matcher::vector_length(this, $src1);
22491 __ kmovwl($tmp$$Register, $src1$$KRegister);
22492 __ andl($tmp$$Register, (1 << masklen) - 1);
22493 __ cmpl($tmp$$Register, (1 << masklen) - 1);
22494 %}
22495 ins_pipe( pipe_slow );
22496 %}
22497
22498 instruct ktest_anytrue_le8(rFlagsRegU cr, kReg src1, kReg src2, rRegI tmp) %{
22499 predicate((Matcher::vector_length(n->in(1)) < 8 ||
22500 (Matcher::vector_length(n->in(1)) == 8 && !VM_Version::supports_avx512dq())) &&
22501 static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
22502 match(Set cr (VectorTest src1 src2));
22503 effect(TEMP tmp);
22504 format %{ "ktest_anytrue_le8 $src1, $src2\t! using $tmp as TEMP" %}
22505 ins_encode %{
22506 uint masklen = Matcher::vector_length(this, $src1);
22507 __ kmovwl($tmp$$Register, $src1$$KRegister);
22508 __ andl($tmp$$Register, (1 << masklen) - 1);
22509 %}
22510 ins_pipe( pipe_slow );
22511 %}
22512
22513 instruct ktest_ge8(rFlagsRegU cr, kReg src1, kReg src2) %{
22514 predicate(Matcher::vector_length(n->in(1)) >= 16 ||
22515 (Matcher::vector_length(n->in(1)) == 8 && VM_Version::supports_avx512dq()));
22516 match(Set cr (VectorTest src1 src2));
22517 format %{ "ktest_ge8 $src1, $src2\n\t" %}
22518 ins_encode %{
22519 uint masklen = Matcher::vector_length(this, $src1);
22520 __ kortest(masklen, $src1$$KRegister, $src1$$KRegister);
22521 %}
22522 ins_pipe( pipe_slow );
22523 %}
22524
22525 //------------------------------------- LoadMask --------------------------------------------
22526
22527 instruct loadMask(legVec dst, legVec src) %{
22528 predicate(n->bottom_type()->isa_pvectmask() == nullptr && !VM_Version::supports_avx512vlbw());
22529 match(Set dst (VectorLoadMask src));
22530 effect(TEMP dst);
22531 format %{ "vector_loadmask_byte $dst, $src\n\t" %}
22532 ins_encode %{
22533 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
22534 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22535 __ load_vector_mask($dst$$XMMRegister, $src$$XMMRegister, vlen_in_bytes, elem_bt, true);
22536 %}
22537 ins_pipe( pipe_slow );
22538 %}
22539
22540 instruct loadMask64(kReg dst, vec src, vec xtmp) %{
22541 predicate(n->bottom_type()->isa_pvectmask() && !VM_Version::supports_avx512vlbw());
22542 match(Set dst (VectorLoadMask src));
22543 effect(TEMP xtmp);
22544 format %{ "vector_loadmask_64byte $dst, $src\t! using $xtmp as TEMP" %}
22545 ins_encode %{
22546 __ load_vector_mask($dst$$KRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
22547 true, Assembler::AVX_512bit);
22548 %}
22549 ins_pipe( pipe_slow );
22550 %}
22551
22552 instruct loadMask_evex(kReg dst, vec src, vec xtmp) %{
22553 predicate(n->bottom_type()->isa_pvectmask() && VM_Version::supports_avx512vlbw());
22554 match(Set dst (VectorLoadMask src));
22555 effect(TEMP xtmp);
22556 format %{ "vector_loadmask_byte $dst, $src\t! using $xtmp as TEMP" %}
22557 ins_encode %{
22558 int vlen_enc = vector_length_encoding(in(1));
22559 __ load_vector_mask($dst$$KRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
22560 false, vlen_enc);
22561 %}
22562 ins_pipe( pipe_slow );
22563 %}
22564
22565 //------------------------------------- StoreMask --------------------------------------------
22566
22567 instruct vstoreMask1B(vec dst, vec src, immI_1 size) %{
22568 predicate(Matcher::vector_length(n) < 64 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22569 match(Set dst (VectorStoreMask src size));
22570 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22571 ins_encode %{
22572 int vlen = Matcher::vector_length(this);
22573 if (vlen <= 16 && UseAVX <= 2) {
22574 assert(UseSSE >= 3, "required");
22575 __ pabsb($dst$$XMMRegister, $src$$XMMRegister);
22576 } else {
22577 assert(UseAVX > 0, "required");
22578 int src_vlen_enc = vector_length_encoding(this, $src);
22579 __ vpabsb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
22580 }
22581 %}
22582 ins_pipe( pipe_slow );
22583 %}
22584
22585 instruct vstoreMask2B(vec dst, vec src, vec xtmp, immI_2 size) %{
22586 predicate(Matcher::vector_length(n) <= 16 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22587 match(Set dst (VectorStoreMask src size));
22588 effect(TEMP_DEF dst, TEMP xtmp);
22589 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22590 ins_encode %{
22591 int vlen_enc = Assembler::AVX_128bit;
22592 int vlen = Matcher::vector_length(this);
22593 if (vlen <= 8) {
22594 assert(UseSSE >= 3, "required");
22595 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
22596 __ pabsw($dst$$XMMRegister, $src$$XMMRegister);
22597 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
22598 } else {
22599 assert(UseAVX > 0, "required");
22600 __ vextracti128($dst$$XMMRegister, $src$$XMMRegister, 0x1);
22601 __ vpacksswb($dst$$XMMRegister, $src$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22602 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22603 }
22604 %}
22605 ins_pipe( pipe_slow );
22606 %}
22607
22608 instruct vstoreMask4B(vec dst, vec src, vec xtmp, immI_4 size) %{
22609 predicate(UseAVX <= 2 && Matcher::vector_length(n) <= 8 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22610 match(Set dst (VectorStoreMask src size));
22611 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22612 effect(TEMP_DEF dst, TEMP xtmp);
22613 ins_encode %{
22614 int vlen_enc = Assembler::AVX_128bit;
22615 int vlen = Matcher::vector_length(this);
22616 if (vlen <= 4) {
22617 assert(UseSSE >= 3, "required");
22618 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
22619 __ pabsd($dst$$XMMRegister, $src$$XMMRegister);
22620 __ packusdw($dst$$XMMRegister, $xtmp$$XMMRegister);
22621 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
22622 } else {
22623 assert(UseAVX > 0, "required");
22624 __ vpxor($xtmp$$XMMRegister, $xtmp$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
22625 __ vextracti128($dst$$XMMRegister, $src$$XMMRegister, 0x1);
22626 __ vpackssdw($dst$$XMMRegister, $src$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22627 __ vpacksswb($dst$$XMMRegister, $dst$$XMMRegister, $xtmp$$XMMRegister, vlen_enc);
22628 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22629 }
22630 %}
22631 ins_pipe( pipe_slow );
22632 %}
22633
22634 instruct storeMask8B(vec dst, vec src, vec xtmp, immI_8 size) %{
22635 predicate(UseAVX <= 2 && Matcher::vector_length(n) == 2);
22636 match(Set dst (VectorStoreMask src size));
22637 effect(TEMP_DEF dst, TEMP xtmp);
22638 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22639 ins_encode %{
22640 assert(UseSSE >= 3, "required");
22641 __ pxor($xtmp$$XMMRegister, $xtmp$$XMMRegister);
22642 __ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x8);
22643 __ pabsd($dst$$XMMRegister, $dst$$XMMRegister);
22644 __ packusdw($dst$$XMMRegister, $xtmp$$XMMRegister);
22645 __ packuswb($dst$$XMMRegister, $xtmp$$XMMRegister);
22646 %}
22647 ins_pipe( pipe_slow );
22648 %}
22649
22650 instruct storeMask8B_avx(vec dst, vec src, immI_8 size, vec vtmp) %{
22651 predicate(UseAVX <= 2 && Matcher::vector_length(n) == 4);
22652 match(Set dst (VectorStoreMask src size));
22653 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s], using $vtmp as TEMP" %}
22654 effect(TEMP_DEF dst, TEMP vtmp);
22655 ins_encode %{
22656 int vlen_enc = Assembler::AVX_128bit;
22657 __ vshufps($dst$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 0x88, Assembler::AVX_256bit);
22658 __ vextracti128($vtmp$$XMMRegister, $dst$$XMMRegister, 0x1);
22659 __ vblendps($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, 0xC, vlen_enc);
22660 __ vpxor($vtmp$$XMMRegister, $vtmp$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22661 __ vpackssdw($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22662 __ vpacksswb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22663 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
22664 %}
22665 ins_pipe( pipe_slow );
22666 %}
22667
22668 instruct vstoreMask4B_evex_novectmask(vec dst, vec src, immI_4 size) %{
22669 predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22670 match(Set dst (VectorStoreMask src size));
22671 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22672 ins_encode %{
22673 int src_vlen_enc = vector_length_encoding(this, $src);
22674 int dst_vlen_enc = vector_length_encoding(this);
22675 if (!VM_Version::supports_avx512vl()) {
22676 src_vlen_enc = Assembler::AVX_512bit;
22677 }
22678 __ evpmovdb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
22679 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
22680 %}
22681 ins_pipe( pipe_slow );
22682 %}
22683
22684 instruct vstoreMask8B_evex_novectmask(vec dst, vec src, immI_8 size) %{
22685 predicate(UseAVX > 2 && n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
22686 match(Set dst (VectorStoreMask src size));
22687 format %{ "vector_store_mask $dst, $src \t! elem size is $size byte[s]" %}
22688 ins_encode %{
22689 int src_vlen_enc = vector_length_encoding(this, $src);
22690 int dst_vlen_enc = vector_length_encoding(this);
22691 if (!VM_Version::supports_avx512vl()) {
22692 src_vlen_enc = Assembler::AVX_512bit;
22693 }
22694 __ evpmovqb($dst$$XMMRegister, $src$$XMMRegister, src_vlen_enc);
22695 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
22696 %}
22697 ins_pipe( pipe_slow );
22698 %}
22699
22700 instruct vstoreMask_evex_vectmask(vec dst, kReg mask, immI size) %{
22701 predicate(n->in(1)->bottom_type()->isa_pvectmask() && !VM_Version::supports_avx512vlbw());
22702 match(Set dst (VectorStoreMask mask size));
22703 effect(TEMP_DEF dst);
22704 format %{ "vector_store_mask $dst, $mask \t! elem size is $size byte[s]" %}
22705 ins_encode %{
22706 assert(Matcher::vector_length_in_bytes(this, $mask) == 64, "");
22707 __ evmovdqul($dst$$XMMRegister, $mask$$KRegister, ExternalAddress(vector_int_mask_cmp_bits()),
22708 false, Assembler::AVX_512bit, noreg);
22709 __ evpmovdb($dst$$XMMRegister, $dst$$XMMRegister, Assembler::AVX_512bit);
22710 %}
22711 ins_pipe( pipe_slow );
22712 %}
22713
22714 instruct vstoreMask_evex(vec dst, kReg mask, immI size) %{
22715 predicate(n->in(1)->bottom_type()->isa_pvectmask() && VM_Version::supports_avx512vlbw());
22716 match(Set dst (VectorStoreMask mask size));
22717 effect(TEMP_DEF dst);
22718 format %{ "vector_store_mask $dst, $mask \t! elem size is $size byte[s]" %}
22719 ins_encode %{
22720 int dst_vlen_enc = vector_length_encoding(this);
22721 __ evpmovm2b($dst$$XMMRegister, $mask$$KRegister, dst_vlen_enc);
22722 __ vpabsb($dst$$XMMRegister, $dst$$XMMRegister, dst_vlen_enc);
22723 %}
22724 ins_pipe( pipe_slow );
22725 %}
22726
22727 instruct vmaskcast_evex(kReg dst) %{
22728 match(Set dst (VectorMaskCast dst));
22729 ins_cost(0);
22730 format %{ "vector_mask_cast $dst" %}
22731 ins_encode %{
22732 // empty
22733 %}
22734 ins_pipe(empty);
22735 %}
22736
22737 instruct vmaskcast(vec dst) %{
22738 predicate(Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
22739 match(Set dst (VectorMaskCast dst));
22740 ins_cost(0);
22741 format %{ "vector_mask_cast $dst" %}
22742 ins_encode %{
22743 // empty
22744 %}
22745 ins_pipe(empty);
22746 %}
22747
22748 instruct vmaskcast_avx(vec dst, vec src) %{
22749 predicate(Matcher::vector_length_in_bytes(n) != Matcher::vector_length_in_bytes(n->in(1)));
22750 match(Set dst (VectorMaskCast src));
22751 format %{ "vector_mask_cast $dst, $src" %}
22752 ins_encode %{
22753 int vlen = Matcher::vector_length(this);
22754 BasicType src_bt = Matcher::vector_element_basic_type(this, $src);
22755 BasicType dst_bt = Matcher::vector_element_basic_type(this);
22756 __ vector_mask_cast($dst$$XMMRegister, $src$$XMMRegister, dst_bt, src_bt, vlen);
22757 %}
22758 ins_pipe(pipe_slow);
22759 %}
22760
22761 //-------------------------------- Load Iota Indices ----------------------------------
22762
22763 instruct loadIotaIndices(vec dst, immI_0 src) %{
22764 match(Set dst (VectorLoadConst src));
22765 format %{ "vector_load_iota $dst CONSTANT_MEMORY\t! load iota indices" %}
22766 ins_encode %{
22767 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
22768 BasicType bt = Matcher::vector_element_basic_type(this);
22769 __ load_iota_indices($dst$$XMMRegister, vlen_in_bytes, bt);
22770 %}
22771 ins_pipe( pipe_slow );
22772 %}
22773
22774 instruct VectorPopulateIndex(vec dst, rRegI src1, immI_1 src2, vec vtmp) %{
22775 match(Set dst (PopulateIndex src1 src2));
22776 effect(TEMP dst, TEMP vtmp);
22777 format %{ "vector_populate_index $dst $src1 $src2\t! using $vtmp as TEMP" %}
22778 ins_encode %{
22779 assert($src2$$constant == 1, "required");
22780 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
22781 int vlen_enc = vector_length_encoding(this);
22782 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22783 __ vpbroadcast(elem_bt, $vtmp$$XMMRegister, $src1$$Register, vlen_enc);
22784 __ load_iota_indices($dst$$XMMRegister, vlen_in_bytes, elem_bt);
22785 __ vpadd(elem_bt, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22786 %}
22787 ins_pipe( pipe_slow );
22788 %}
22789
22790 instruct VectorPopulateLIndex(vec dst, rRegL src1, immI_1 src2, vec vtmp) %{
22791 match(Set dst (PopulateIndex src1 src2));
22792 effect(TEMP dst, TEMP vtmp);
22793 format %{ "vector_populate_index $dst $src1 $src2\t! using $vtmp as TEMP" %}
22794 ins_encode %{
22795 assert($src2$$constant == 1, "required");
22796 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
22797 int vlen_enc = vector_length_encoding(this);
22798 BasicType elem_bt = Matcher::vector_element_basic_type(this);
22799 __ vpbroadcast(elem_bt, $vtmp$$XMMRegister, $src1$$Register, vlen_enc);
22800 __ load_iota_indices($dst$$XMMRegister, vlen_in_bytes, elem_bt);
22801 __ vpadd(elem_bt, $dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22802 %}
22803 ins_pipe( pipe_slow );
22804 %}
22805
22806 //-------------------------------- Rearrange ----------------------------------
22807
22808 // LoadShuffle/Rearrange for Byte
22809 instruct rearrangeB(vec dst, vec shuffle) %{
22810 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
22811 Matcher::vector_length(n) < 32);
22812 match(Set dst (VectorRearrange dst shuffle));
22813 format %{ "vector_rearrange $dst, $shuffle, $dst" %}
22814 ins_encode %{
22815 assert(UseSSE >= 4, "required");
22816 __ pshufb($dst$$XMMRegister, $shuffle$$XMMRegister);
22817 %}
22818 ins_pipe( pipe_slow );
22819 %}
22820
22821 instruct rearrangeB_avx(legVec dst, legVec src, vec shuffle, legVec vtmp1, legVec vtmp2) %{
22822 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
22823 Matcher::vector_length(n) == 32 && !VM_Version::supports_avx512_vbmi());
22824 match(Set dst (VectorRearrange src shuffle));
22825 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
22826 format %{ "vector_rearrange $dst, $shuffle, $src\t! using $vtmp1, $vtmp2 as TEMP" %}
22827 ins_encode %{
22828 assert(UseAVX >= 2, "required");
22829 // Swap src into vtmp1
22830 __ vperm2i128($vtmp1$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 1);
22831 // Shuffle swapped src to get entries from other 128 bit lane
22832 __ vpshufb($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
22833 // Shuffle original src to get entries from self 128 bit lane
22834 __ vpshufb($dst$$XMMRegister, $src$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
22835 // Create a blend mask by setting high bits for entries coming from other lane in shuffle
22836 __ vpaddb($vtmp2$$XMMRegister, $shuffle$$XMMRegister, ExternalAddress(vector_byte_shufflemask()), Assembler::AVX_256bit, noreg);
22837 // Perform the blend
22838 __ vpblendvb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, Assembler::AVX_256bit);
22839 %}
22840 ins_pipe( pipe_slow );
22841 %}
22842
22843
22844 instruct rearrangeB_evex(vec dst, vec src, vec shuffle, vec xtmp1, vec xtmp2, vec xtmp3, kReg ktmp, rRegI rtmp) %{
22845 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
22846 Matcher::vector_length(n) > 32 && !VM_Version::supports_avx512_vbmi());
22847 match(Set dst (VectorRearrange src shuffle));
22848 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP ktmp, TEMP rtmp);
22849 format %{ "vector_rearrange $dst, $shuffle, $src!\t using $xtmp1, $xtmp2, $xtmp3, $rtmp and $ktmp as TEMP" %}
22850 ins_encode %{
22851 int vlen_enc = vector_length_encoding(this);
22852 __ rearrange_bytes($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister,
22853 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $xtmp3$$XMMRegister,
22854 $rtmp$$Register, $ktmp$$KRegister, vlen_enc);
22855 %}
22856 ins_pipe( pipe_slow );
22857 %}
22858
22859 instruct rearrangeB_evex_vbmi(vec dst, vec src, vec shuffle) %{
22860 predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
22861 Matcher::vector_length(n) >= 32 && VM_Version::supports_avx512_vbmi());
22862 match(Set dst (VectorRearrange src shuffle));
22863 format %{ "vector_rearrange $dst, $shuffle, $src" %}
22864 ins_encode %{
22865 int vlen_enc = vector_length_encoding(this);
22866 __ vpermb($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
22867 %}
22868 ins_pipe( pipe_slow );
22869 %}
22870
22871 // LoadShuffle/Rearrange for Short
22872
22873 instruct loadShuffleS(vec dst, vec src, vec vtmp) %{
22874 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
22875 !VM_Version::supports_avx512bw());
22876 match(Set dst (VectorLoadShuffle src));
22877 effect(TEMP dst, TEMP vtmp);
22878 format %{ "vector_load_shuffle $dst, $src\t! using $vtmp as TEMP" %}
22879 ins_encode %{
22880 // Create a byte shuffle mask from short shuffle mask
22881 // only byte shuffle instruction available on these platforms
22882 int vlen_in_bytes = Matcher::vector_length_in_bytes(this);
22883 if (UseAVX == 0) {
22884 assert(vlen_in_bytes <= 16, "required");
22885 // Multiply each shuffle by two to get byte index
22886 __ movdqu($vtmp$$XMMRegister, $src$$XMMRegister);
22887 __ psllw($vtmp$$XMMRegister, 1);
22888
22889 // Duplicate to create 2 copies of byte index
22890 __ movdqu($dst$$XMMRegister, $vtmp$$XMMRegister);
22891 __ psllw($dst$$XMMRegister, 8);
22892 __ por($dst$$XMMRegister, $vtmp$$XMMRegister);
22893
22894 // Add one to get alternate byte index
22895 __ movdqu($vtmp$$XMMRegister, ExternalAddress(vector_short_shufflemask()), noreg);
22896 __ paddb($dst$$XMMRegister, $vtmp$$XMMRegister);
22897 } else {
22898 assert(UseAVX > 1 || vlen_in_bytes <= 16, "required");
22899 int vlen_enc = vector_length_encoding(this);
22900 // Multiply each shuffle by two to get byte index
22901 __ vpsllw($vtmp$$XMMRegister, $src$$XMMRegister, 1, vlen_enc);
22902
22903 // Duplicate to create 2 copies of byte index
22904 __ vpsllw($dst$$XMMRegister, $vtmp$$XMMRegister, 8, vlen_enc);
22905 __ vpor($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
22906
22907 // Add one to get alternate byte index
22908 __ vpaddb($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_short_shufflemask()), vlen_enc, noreg);
22909 }
22910 %}
22911 ins_pipe( pipe_slow );
22912 %}
22913
22914 instruct rearrangeS(vec dst, vec shuffle) %{
22915 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
22916 Matcher::vector_length(n) <= 8 && !VM_Version::supports_avx512bw());
22917 match(Set dst (VectorRearrange dst shuffle));
22918 format %{ "vector_rearrange $dst, $shuffle, $dst" %}
22919 ins_encode %{
22920 assert(UseSSE >= 4, "required");
22921 __ pshufb($dst$$XMMRegister, $shuffle$$XMMRegister);
22922 %}
22923 ins_pipe( pipe_slow );
22924 %}
22925
22926 instruct rearrangeS_avx(legVec dst, legVec src, vec shuffle, legVec vtmp1, legVec vtmp2) %{
22927 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
22928 Matcher::vector_length(n) == 16 && !VM_Version::supports_avx512bw());
22929 match(Set dst (VectorRearrange src shuffle));
22930 effect(TEMP dst, TEMP vtmp1, TEMP vtmp2);
22931 format %{ "vector_rearrange $dst, $shuffle, $src\t! using $vtmp1, $vtmp2 as TEMP" %}
22932 ins_encode %{
22933 assert(UseAVX >= 2, "required");
22934 // Swap src into vtmp1
22935 __ vperm2i128($vtmp1$$XMMRegister, $src$$XMMRegister, $src$$XMMRegister, 1);
22936 // Shuffle swapped src to get entries from other 128 bit lane
22937 __ vpshufb($vtmp1$$XMMRegister, $vtmp1$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
22938 // Shuffle original src to get entries from self 128 bit lane
22939 __ vpshufb($dst$$XMMRegister, $src$$XMMRegister, $shuffle$$XMMRegister, Assembler::AVX_256bit);
22940 // Create a blend mask by setting high bits for entries coming from other lane in shuffle
22941 __ vpaddb($vtmp2$$XMMRegister, $shuffle$$XMMRegister, ExternalAddress(vector_byte_shufflemask()), Assembler::AVX_256bit, noreg);
22942 // Perform the blend
22943 __ vpblendvb($dst$$XMMRegister, $dst$$XMMRegister, $vtmp1$$XMMRegister, $vtmp2$$XMMRegister, Assembler::AVX_256bit);
22944 %}
22945 ins_pipe( pipe_slow );
22946 %}
22947
22948 instruct rearrangeS_evex(vec dst, vec src, vec shuffle) %{
22949 predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
22950 VM_Version::supports_avx512bw());
22951 match(Set dst (VectorRearrange src shuffle));
22952 format %{ "vector_rearrange $dst, $shuffle, $src" %}
22953 ins_encode %{
22954 int vlen_enc = vector_length_encoding(this);
22955 if (!VM_Version::supports_avx512vl()) {
22956 vlen_enc = Assembler::AVX_512bit;
22957 }
22958 __ vpermw($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
22959 %}
22960 ins_pipe( pipe_slow );
22961 %}
22962
22963 // LoadShuffle/Rearrange for Integer and Float
22964
22965 instruct loadShuffleI(vec dst, vec src, vec vtmp) %{
22966 predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
22967 Matcher::vector_length(n) == 4 && UseAVX == 0);
22968 match(Set dst (VectorLoadShuffle src));
22969 effect(TEMP dst, TEMP vtmp);
22970 format %{ "vector_load_shuffle $dst, $src\t! using $vtmp as TEMP" %}
22971 ins_encode %{
22972 assert(UseSSE >= 4, "required");
22973
22974 // Create a byte shuffle mask from int shuffle mask
22975 // only byte shuffle instruction available on these platforms
22976
22977 // Duplicate and multiply each shuffle by 4
22978 __ movdqu($vtmp$$XMMRegister, $src$$XMMRegister);
22979 __ pshuflw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 0xA0);
22980 __ pshufhw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 0xA0);
22981 __ psllw($vtmp$$XMMRegister, 2);
22982
22983 // Duplicate again to create 4 copies of byte index
22984 __ movdqu($dst$$XMMRegister, $vtmp$$XMMRegister);
22985 __ psllw($dst$$XMMRegister, 8);
22986 __ por($vtmp$$XMMRegister, $dst$$XMMRegister);
22987
22988 // Add 3,2,1,0 to get alternate byte index
22989 __ movdqu($dst$$XMMRegister, ExternalAddress(vector_int_shufflemask()), noreg);
22990 __ paddb($dst$$XMMRegister, $vtmp$$XMMRegister);
22991 %}
22992 ins_pipe( pipe_slow );
22993 %}
22994
22995 instruct rearrangeI(vec dst, vec shuffle) %{
22996 predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
22997 UseAVX == 0);
22998 match(Set dst (VectorRearrange dst shuffle));
22999 format %{ "vector_rearrange $dst, $shuffle, $dst" %}
23000 ins_encode %{
23001 assert(UseSSE >= 4, "required");
23002 __ pshufb($dst$$XMMRegister, $shuffle$$XMMRegister);
23003 %}
23004 ins_pipe( pipe_slow );
23005 %}
23006
23007 instruct rearrangeI_avx(vec dst, vec src, vec shuffle) %{
23008 predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
23009 UseAVX > 0);
23010 match(Set dst (VectorRearrange src shuffle));
23011 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23012 ins_encode %{
23013 int vlen_enc = vector_length_encoding(this);
23014 BasicType bt = Matcher::vector_element_basic_type(this);
23015 __ vector_rearrange_int_float(bt, $dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23016 %}
23017 ins_pipe( pipe_slow );
23018 %}
23019
23020 // LoadShuffle/Rearrange for Long and Double
23021
23022 instruct loadShuffleL(vec dst, vec src, vec vtmp) %{
23023 predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
23024 Matcher::vector_length(n) < 8 && !VM_Version::supports_avx512vl());
23025 match(Set dst (VectorLoadShuffle src));
23026 effect(TEMP dst, TEMP vtmp);
23027 format %{ "vector_load_shuffle $dst, $src\t! using $vtmp as TEMP" %}
23028 ins_encode %{
23029 assert(UseAVX >= 2, "required");
23030
23031 int vlen_enc = vector_length_encoding(this);
23032 // Create a double word shuffle mask from long shuffle mask
23033 // only double word shuffle instruction available on these platforms
23034
23035 // Multiply each shuffle by two to get double word index
23036 __ vpsllq($vtmp$$XMMRegister, $src$$XMMRegister, 1, vlen_enc);
23037
23038 // Duplicate each double word shuffle
23039 __ vpsllq($dst$$XMMRegister, $vtmp$$XMMRegister, 32, vlen_enc);
23040 __ vpor($dst$$XMMRegister, $dst$$XMMRegister, $vtmp$$XMMRegister, vlen_enc);
23041
23042 // Add one to get alternate double word index
23043 __ vpaddd($dst$$XMMRegister, $dst$$XMMRegister, ExternalAddress(vector_long_shufflemask()), vlen_enc, noreg);
23044 %}
23045 ins_pipe( pipe_slow );
23046 %}
23047
23048 instruct rearrangeL(vec dst, vec src, vec shuffle) %{
23049 predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
23050 Matcher::vector_length(n) < 8 && !VM_Version::supports_avx512vl());
23051 match(Set dst (VectorRearrange src shuffle));
23052 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23053 ins_encode %{
23054 assert(UseAVX >= 2, "required");
23055
23056 int vlen_enc = vector_length_encoding(this);
23057 __ vpermd($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23058 %}
23059 ins_pipe( pipe_slow );
23060 %}
23061
23062 instruct rearrangeL_evex(vec dst, vec src, vec shuffle) %{
23063 predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
23064 (Matcher::vector_length(n) == 8 || VM_Version::supports_avx512vl()));
23065 match(Set dst (VectorRearrange src shuffle));
23066 format %{ "vector_rearrange $dst, $shuffle, $src" %}
23067 ins_encode %{
23068 assert(UseAVX > 2, "required");
23069
23070 int vlen_enc = vector_length_encoding(this);
23071 if (vlen_enc == Assembler::AVX_128bit) {
23072 vlen_enc = Assembler::AVX_256bit;
23073 }
23074 __ vpermq($dst$$XMMRegister, $shuffle$$XMMRegister, $src$$XMMRegister, vlen_enc);
23075 %}
23076 ins_pipe( pipe_slow );
23077 %}
23078
23079 // --------------------------------- FMA --------------------------------------
23080 // a * b + c
23081
23082 instruct vfmaF_reg(vec a, vec b, vec c) %{
23083 match(Set c (FmaVF c (Binary a b)));
23084 format %{ "fmaps $a,$b,$c\t# $c = $a * $b + $c fma packedF" %}
23085 ins_cost(150);
23086 ins_encode %{
23087 assert(UseFMA, "not enabled");
23088 int vlen_enc = vector_length_encoding(this);
23089 __ vfmaf($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister, vlen_enc);
23090 %}
23091 ins_pipe( pipe_slow );
23092 %}
23093
23094 instruct vfmaF_mem(vec a, memory b, vec c) %{
23095 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
23096 match(Set c (FmaVF c (Binary a (LoadVector b))));
23097 format %{ "fmaps $a,$b,$c\t# $c = $a * $b + $c fma packedF" %}
23098 ins_cost(150);
23099 ins_encode %{
23100 assert(UseFMA, "not enabled");
23101 int vlen_enc = vector_length_encoding(this);
23102 __ vfmaf($c$$XMMRegister, $a$$XMMRegister, $b$$Address, $c$$XMMRegister, vlen_enc);
23103 %}
23104 ins_pipe( pipe_slow );
23105 %}
23106
23107 instruct vfmaD_reg(vec a, vec b, vec c) %{
23108 match(Set c (FmaVD c (Binary a b)));
23109 format %{ "fmapd $a,$b,$c\t# $c = $a * $b + $c fma packedD" %}
23110 ins_cost(150);
23111 ins_encode %{
23112 assert(UseFMA, "not enabled");
23113 int vlen_enc = vector_length_encoding(this);
23114 __ vfmad($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $c$$XMMRegister, vlen_enc);
23115 %}
23116 ins_pipe( pipe_slow );
23117 %}
23118
23119 instruct vfmaD_mem(vec a, memory b, vec c) %{
23120 predicate(Matcher::vector_length_in_bytes(n->in(1)) > 8);
23121 match(Set c (FmaVD c (Binary a (LoadVector b))));
23122 format %{ "fmapd $a,$b,$c\t# $c = $a * $b + $c fma packedD" %}
23123 ins_cost(150);
23124 ins_encode %{
23125 assert(UseFMA, "not enabled");
23126 int vlen_enc = vector_length_encoding(this);
23127 __ vfmad($c$$XMMRegister, $a$$XMMRegister, $b$$Address, $c$$XMMRegister, vlen_enc);
23128 %}
23129 ins_pipe( pipe_slow );
23130 %}
23131
23132 // --------------------------------- Vector Multiply Add --------------------------------------
23133
23134 instruct vmuladdS2I_reg_sse(vec dst, vec src1) %{
23135 predicate(UseAVX == 0);
23136 match(Set dst (MulAddVS2VI dst src1));
23137 format %{ "pmaddwd $dst,$src1\t! muladd packedStoI" %}
23138 ins_encode %{
23139 __ pmaddwd($dst$$XMMRegister, $src1$$XMMRegister);
23140 %}
23141 ins_pipe( pipe_slow );
23142 %}
23143
23144 instruct vmuladdS2I_reg_avx(vec dst, vec src1, vec src2) %{
23145 predicate(UseAVX > 0);
23146 match(Set dst (MulAddVS2VI src1 src2));
23147 format %{ "vpmaddwd $dst,$src1,$src2\t! muladd packedStoI" %}
23148 ins_encode %{
23149 int vlen_enc = vector_length_encoding(this);
23150 __ vpmaddwd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
23151 %}
23152 ins_pipe( pipe_slow );
23153 %}
23154
23155 // --------------------------------- Vector Multiply Add Add ----------------------------------
23156
23157 instruct vmuladdaddS2I_reg(vec dst, vec src1, vec src2) %{
23158 predicate(VM_Version::supports_avx512_vnni());
23159 match(Set dst (AddVI (MulAddVS2VI src1 src2) dst));
23160 format %{ "evpdpwssd $dst,$src1,$src2\t! muladdadd packedStoI" %}
23161 ins_encode %{
23162 assert(UseAVX > 2, "required");
23163 int vlen_enc = vector_length_encoding(this);
23164 __ evpdpwssd($dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
23165 %}
23166 ins_pipe( pipe_slow );
23167 ins_cost(10);
23168 %}
23169
23170 // --------------------------------- PopCount --------------------------------------
23171
23172 instruct vpopcount_integral_reg_evex(vec dst, vec src) %{
23173 predicate(is_vector_popcount_predicate(Matcher::vector_element_basic_type(n->in(1))));
23174 match(Set dst (PopCountVI src));
23175 match(Set dst (PopCountVL src));
23176 format %{ "vector_popcount_integral $dst, $src" %}
23177 ins_encode %{
23178 int opcode = this->ideal_Opcode();
23179 int vlen_enc = vector_length_encoding(this, $src);
23180 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23181 __ vector_popcount_integral_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, k0, true, vlen_enc);
23182 %}
23183 ins_pipe( pipe_slow );
23184 %}
23185
23186 instruct vpopcount_integral_reg_evex_masked(vec dst, vec src, kReg mask) %{
23187 predicate(is_vector_popcount_predicate(Matcher::vector_element_basic_type(n->in(1))));
23188 match(Set dst (PopCountVI src mask));
23189 match(Set dst (PopCountVL src mask));
23190 format %{ "vector_popcount_integral_masked $dst, $src, $mask" %}
23191 ins_encode %{
23192 int vlen_enc = vector_length_encoding(this, $src);
23193 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23194 __ evmovdquq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
23195 __ vector_popcount_integral_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $mask$$KRegister, true, vlen_enc);
23196 %}
23197 ins_pipe( pipe_slow );
23198 %}
23199
23200 instruct vpopcount_avx_reg(vec dst, vec src, vec xtmp1, vec xtmp2, rRegP rtmp) %{
23201 predicate(!is_vector_popcount_predicate(Matcher::vector_element_basic_type(n->in(1))));
23202 match(Set dst (PopCountVI src));
23203 match(Set dst (PopCountVL src));
23204 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp);
23205 format %{ "vector_popcount_integral $dst, $src\t! using $xtmp1, $xtmp2, and $rtmp as TEMP" %}
23206 ins_encode %{
23207 int opcode = this->ideal_Opcode();
23208 int vlen_enc = vector_length_encoding(this, $src);
23209 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23210 __ vector_popcount_integral(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23211 $xtmp2$$XMMRegister, $rtmp$$Register, vlen_enc);
23212 %}
23213 ins_pipe( pipe_slow );
23214 %}
23215
23216 // --------------------------------- Vector Trailing Zeros Count --------------------------------------
23217
23218 instruct vcount_trailing_zeros_reg_evex(vec dst, vec src, vec xtmp, rRegP rtmp) %{
23219 predicate(is_clz_non_subword_predicate_evex(Matcher::vector_element_basic_type(n->in(1)),
23220 Matcher::vector_length_in_bytes(n->in(1))));
23221 match(Set dst (CountTrailingZerosV src));
23222 effect(TEMP dst, TEMP xtmp, TEMP rtmp);
23223 ins_cost(400);
23224 format %{ "vector_count_trailing_zeros $dst, $src!\t using $xtmp and $rtmp as TEMP" %}
23225 ins_encode %{
23226 int vlen_enc = vector_length_encoding(this, $src);
23227 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23228 __ vector_count_trailing_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, xnoreg,
23229 xnoreg, xnoreg, $xtmp$$XMMRegister, k0, $rtmp$$Register, vlen_enc);
23230 %}
23231 ins_pipe( pipe_slow );
23232 %}
23233
23234 instruct vcount_trailing_zeros_short_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, rRegP rtmp) %{
23235 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_SHORT &&
23236 VM_Version::supports_avx512cd() &&
23237 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64));
23238 match(Set dst (CountTrailingZerosV src));
23239 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp);
23240 ins_cost(400);
23241 format %{ "vector_count_trailing_zeros $dst, $src!\t using $xtmp1, $xtmp2, $xtmp3 and $rtmp as TEMP" %}
23242 ins_encode %{
23243 int vlen_enc = vector_length_encoding(this, $src);
23244 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23245 __ vector_count_trailing_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23246 $xtmp2$$XMMRegister, xnoreg, $xtmp3$$XMMRegister, k0, $rtmp$$Register, vlen_enc);
23247 %}
23248 ins_pipe( pipe_slow );
23249 %}
23250
23251 instruct vcount_trailing_zeros_byte_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, kReg ktmp, rRegP rtmp) %{
23252 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_BYTE && VM_Version::supports_avx512vlbw());
23253 match(Set dst (CountTrailingZerosV src));
23254 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, TEMP ktmp, TEMP rtmp);
23255 ins_cost(400);
23256 format %{ "vector_count_trailing_zeros $dst, $src!\t using $xtmp1, $xtmp2, $xtmp3, $xtmp4, $ktmp and $rtmp as TEMP" %}
23257 ins_encode %{
23258 int vlen_enc = vector_length_encoding(this, $src);
23259 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23260 __ vector_count_trailing_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23261 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister,
23262 $ktmp$$KRegister, $rtmp$$Register, vlen_enc);
23263 %}
23264 ins_pipe( pipe_slow );
23265 %}
23266
23267 instruct vcount_trailing_zeros_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, rRegP rtmp) %{
23268 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n->in(1)) < 64);
23269 match(Set dst (CountTrailingZerosV src));
23270 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp);
23271 format %{ "vector_count_trailing_zeros $dst, $src\t! using $xtmp1, $xtmp2, $xtmp3, and $rtmp as TEMP" %}
23272 ins_encode %{
23273 int vlen_enc = vector_length_encoding(this, $src);
23274 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23275 __ vector_count_trailing_zeros_avx(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23276 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, vlen_enc);
23277 %}
23278 ins_pipe( pipe_slow );
23279 %}
23280
23281
23282 // --------------------------------- Bitwise Ternary Logic ----------------------------------
23283
23284 instruct vpternlog(vec dst, vec src2, vec src3, immU8 func) %{
23285 match(Set dst (MacroLogicV (Binary dst src2) (Binary src3 func)));
23286 effect(TEMP dst);
23287 format %{ "vpternlogd $dst,$src2,$src3,$func\t! vector ternary logic" %}
23288 ins_encode %{
23289 int vector_len = vector_length_encoding(this);
23290 __ vpternlogd($dst$$XMMRegister, $func$$constant, $src2$$XMMRegister, $src3$$XMMRegister, vector_len);
23291 %}
23292 ins_pipe( pipe_slow );
23293 %}
23294
23295 instruct vpternlog_mem(vec dst, vec src2, memory src3, immU8 func) %{
23296 predicate(Matcher::vector_length_in_bytes(n->in(1)->in(1)) > 8);
23297 match(Set dst (MacroLogicV (Binary dst src2) (Binary (LoadVector src3) func)));
23298 effect(TEMP dst);
23299 format %{ "vpternlogd $dst,$src2,$src3,$func\t! vector ternary logic" %}
23300 ins_encode %{
23301 int vector_len = vector_length_encoding(this);
23302 __ vpternlogd($dst$$XMMRegister, $func$$constant, $src2$$XMMRegister, $src3$$Address, vector_len);
23303 %}
23304 ins_pipe( pipe_slow );
23305 %}
23306
23307 // --------------------------------- Rotation Operations ----------------------------------
23308 instruct vprotate_immI8(vec dst, vec src, immI8 shift) %{
23309 match(Set dst (RotateLeftV src shift));
23310 match(Set dst (RotateRightV src shift));
23311 format %{ "vprotate_imm8 $dst,$src,$shift\t! vector rotate" %}
23312 ins_encode %{
23313 int opcode = this->ideal_Opcode();
23314 int vector_len = vector_length_encoding(this);
23315 BasicType etype = this->bottom_type()->is_vect()->element_basic_type();
23316 __ vprotate_imm(opcode, etype, $dst$$XMMRegister, $src$$XMMRegister, $shift$$constant, vector_len);
23317 %}
23318 ins_pipe( pipe_slow );
23319 %}
23320
23321 instruct vprorate(vec dst, vec src, vec shift) %{
23322 match(Set dst (RotateLeftV src shift));
23323 match(Set dst (RotateRightV src shift));
23324 format %{ "vprotate $dst,$src,$shift\t! vector rotate" %}
23325 ins_encode %{
23326 int opcode = this->ideal_Opcode();
23327 int vector_len = vector_length_encoding(this);
23328 BasicType etype = this->bottom_type()->is_vect()->element_basic_type();
23329 __ vprotate_var(opcode, etype, $dst$$XMMRegister, $src$$XMMRegister, $shift$$XMMRegister, vector_len);
23330 %}
23331 ins_pipe( pipe_slow );
23332 %}
23333
23334 // ---------------------------------- Masked Operations ------------------------------------
23335 instruct vmasked_load_avx_non_subword(vec dst, memory mem, vec mask) %{
23336 predicate(!n->in(3)->bottom_type()->isa_pvectmask());
23337 match(Set dst (LoadVectorMasked mem mask));
23338 format %{ "vector_masked_load $dst, $mem, $mask \t! vector masked copy" %}
23339 ins_encode %{
23340 BasicType elmType = this->bottom_type()->is_vect()->element_basic_type();
23341 int vlen_enc = vector_length_encoding(this);
23342 __ vmovmask(elmType, $dst$$XMMRegister, $mem$$Address, $mask$$XMMRegister, vlen_enc);
23343 %}
23344 ins_pipe( pipe_slow );
23345 %}
23346
23347
23348 instruct vmasked_load_evex(vec dst, memory mem, kReg mask) %{
23349 predicate(n->in(3)->bottom_type()->isa_pvectmask());
23350 match(Set dst (LoadVectorMasked mem mask));
23351 format %{ "vector_masked_load $dst, $mem, $mask \t! vector masked copy" %}
23352 ins_encode %{
23353 BasicType elmType = this->bottom_type()->is_vect()->element_basic_type();
23354 int vector_len = vector_length_encoding(this);
23355 __ evmovdqu(elmType, $mask$$KRegister, $dst$$XMMRegister, $mem$$Address, false, vector_len);
23356 %}
23357 ins_pipe( pipe_slow );
23358 %}
23359
23360 instruct vmasked_store_avx_non_subword(memory mem, vec src, vec mask) %{
23361 predicate(!n->in(3)->in(2)->bottom_type()->isa_pvectmask());
23362 match(Set mem (StoreVectorMasked mem (Binary src mask)));
23363 format %{ "vector_masked_store $mem, $src, $mask \t! vector masked store" %}
23364 ins_encode %{
23365 const MachNode* src_node = static_cast<const MachNode*>(this->in(this->operand_index($src)));
23366 int vlen_enc = vector_length_encoding(src_node);
23367 BasicType elmType = src_node->bottom_type()->is_vect()->element_basic_type();
23368 __ vmovmask(elmType, $mem$$Address, $src$$XMMRegister, $mask$$XMMRegister, vlen_enc);
23369 %}
23370 ins_pipe( pipe_slow );
23371 %}
23372
23373 instruct vmasked_store_evex(memory mem, vec src, kReg mask) %{
23374 predicate(n->in(3)->in(2)->bottom_type()->isa_pvectmask());
23375 match(Set mem (StoreVectorMasked mem (Binary src mask)));
23376 format %{ "vector_masked_store $mem, $src, $mask \t! vector masked store" %}
23377 ins_encode %{
23378 const MachNode* src_node = static_cast<const MachNode*>(this->in(this->operand_index($src)));
23379 BasicType elmType = src_node->bottom_type()->is_vect()->element_basic_type();
23380 int vlen_enc = vector_length_encoding(src_node);
23381 __ evmovdqu(elmType, $mask$$KRegister, $mem$$Address, $src$$XMMRegister, true, vlen_enc);
23382 %}
23383 ins_pipe( pipe_slow );
23384 %}
23385
23386 instruct verify_vector_alignment(rRegP addr, immL32 mask, rFlagsReg cr) %{
23387 match(Set addr (VerifyVectorAlignment addr mask));
23388 effect(KILL cr);
23389 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
23390 ins_encode %{
23391 Label Lskip;
23392 // check if masked bits of addr are zero
23393 __ testq($addr$$Register, $mask$$constant);
23394 __ jccb(Assembler::equal, Lskip);
23395 __ stop("verify_vector_alignment found a misaligned vector memory access");
23396 __ bind(Lskip);
23397 %}
23398 ins_pipe(pipe_slow);
23399 %}
23400
23401 instruct vmask_cmp_node(rRegI dst, vec src1, vec src2, kReg mask, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
23402 match(Set dst (VectorCmpMasked src1 (Binary src2 mask)));
23403 effect(TEMP_DEF dst, TEMP ktmp1, TEMP ktmp2, KILL cr);
23404 format %{ "vector_mask_cmp $src1, $src2, $mask \t! vector mask comparison" %}
23405 ins_encode %{
23406 assert(vector_length_encoding(this, $src1) == vector_length_encoding(this, $src2), "mismatch");
23407 assert(Matcher::vector_element_basic_type(this, $src1) == Matcher::vector_element_basic_type(this, $src2), "mismatch");
23408
23409 Label DONE;
23410 int vlen_enc = vector_length_encoding(this, $src1);
23411 BasicType elem_bt = Matcher::vector_element_basic_type(this, $src1);
23412
23413 __ knotql($ktmp2$$KRegister, $mask$$KRegister);
23414 __ mov64($dst$$Register, -1L);
23415 __ evpcmp(elem_bt, $ktmp1$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, Assembler::eq, vlen_enc);
23416 __ kortestql($ktmp2$$KRegister, $ktmp1$$KRegister);
23417 __ jccb(Assembler::carrySet, DONE);
23418 __ kmovql($dst$$Register, $ktmp1$$KRegister);
23419 __ notq($dst$$Register);
23420 __ tzcntq($dst$$Register, $dst$$Register);
23421 __ bind(DONE);
23422 %}
23423 ins_pipe( pipe_slow );
23424 %}
23425
23426
23427 instruct vmask_gen(kReg dst, rRegL len, rRegL temp, rFlagsReg cr) %{
23428 match(Set dst (VectorMaskGen len));
23429 effect(TEMP temp, KILL cr);
23430 format %{ "vector_mask_gen32 $dst, $len \t! vector mask generator" %}
23431 ins_encode %{
23432 __ genmask($dst$$KRegister, $len$$Register, $temp$$Register);
23433 %}
23434 ins_pipe( pipe_slow );
23435 %}
23436
23437 instruct vmask_gen_imm(kReg dst, immL len, rRegL temp) %{
23438 match(Set dst (VectorMaskGen len));
23439 format %{ "vector_mask_gen $len \t! vector mask generator" %}
23440 effect(TEMP temp);
23441 ins_encode %{
23442 if ($len$$constant > 0) {
23443 __ mov64($temp$$Register, right_n_bits($len$$constant));
23444 __ kmovql($dst$$KRegister, $temp$$Register);
23445 } else {
23446 __ kxorql($dst$$KRegister, $dst$$KRegister, $dst$$KRegister);
23447 }
23448 %}
23449 ins_pipe( pipe_slow );
23450 %}
23451
23452 instruct vmask_tolong_evex(rRegL dst, kReg mask, rFlagsReg cr) %{
23453 predicate(n->in(1)->bottom_type()->isa_pvectmask());
23454 match(Set dst (VectorMaskToLong mask));
23455 effect(TEMP dst, KILL cr);
23456 format %{ "vector_tolong_evex $dst, $mask \t! vector mask tolong" %}
23457 ins_encode %{
23458 int opcode = this->ideal_Opcode();
23459 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23460 int mask_len = Matcher::vector_length(this, $mask);
23461 int mask_size = mask_len * type2aelembytes(mbt);
23462 int vlen_enc = vector_length_encoding(this, $mask);
23463 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
23464 $dst$$Register, mask_len, mask_size, vlen_enc);
23465 %}
23466 ins_pipe( pipe_slow );
23467 %}
23468
23469 instruct vmask_tolong_bool(rRegL dst, vec mask, vec xtmp, rFlagsReg cr) %{
23470 predicate(n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23471 match(Set dst (VectorMaskToLong mask));
23472 format %{ "vector_tolong_bool $dst, $mask \t! using $xtmp as TEMP" %}
23473 effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
23474 ins_encode %{
23475 int opcode = this->ideal_Opcode();
23476 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23477 int mask_len = Matcher::vector_length(this, $mask);
23478 int vlen_enc = vector_length_encoding(this, $mask);
23479 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23480 $dst$$Register, mask_len, mbt, vlen_enc);
23481 %}
23482 ins_pipe( pipe_slow );
23483 %}
23484
23485 instruct vmask_tolong_avx(rRegL dst, vec mask, immI size, vec xtmp, rFlagsReg cr) %{
23486 predicate(n->in(1)->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23487 match(Set dst (VectorMaskToLong (VectorStoreMask mask size)));
23488 format %{ "vector_tolong_avx $dst, $mask \t! using $xtmp as TEMP" %}
23489 effect(TEMP_DEF dst, TEMP xtmp, KILL cr);
23490 ins_encode %{
23491 int opcode = this->ideal_Opcode();
23492 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23493 int mask_len = Matcher::vector_length(this, $mask);
23494 int vlen_enc = vector_length_encoding(this, $mask);
23495 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23496 $dst$$Register, mask_len, mbt, vlen_enc);
23497 %}
23498 ins_pipe( pipe_slow );
23499 %}
23500
23501 instruct vmask_truecount_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
23502 predicate(n->in(1)->bottom_type()->isa_pvectmask());
23503 match(Set dst (VectorMaskTrueCount mask));
23504 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
23505 format %{ "vector_truecount_evex $dst, $mask \t! using $tmp as TEMP" %}
23506 ins_encode %{
23507 int opcode = this->ideal_Opcode();
23508 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23509 int mask_len = Matcher::vector_length(this, $mask);
23510 int mask_size = mask_len * type2aelembytes(mbt);
23511 int vlen_enc = vector_length_encoding(this, $mask);
23512 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
23513 $tmp$$Register, mask_len, mask_size, vlen_enc);
23514 %}
23515 ins_pipe( pipe_slow );
23516 %}
23517
23518 instruct vmask_truecount_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23519 predicate(n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23520 match(Set dst (VectorMaskTrueCount mask));
23521 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23522 format %{ "vector_truecount_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23523 ins_encode %{
23524 int opcode = this->ideal_Opcode();
23525 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23526 int mask_len = Matcher::vector_length(this, $mask);
23527 int vlen_enc = vector_length_encoding(this, $mask);
23528 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23529 $tmp$$Register, mask_len, mbt, vlen_enc);
23530 %}
23531 ins_pipe( pipe_slow );
23532 %}
23533
23534 instruct vmask_truecount_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23535 predicate(n->in(1)->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23536 match(Set dst (VectorMaskTrueCount (VectorStoreMask mask size)));
23537 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23538 format %{ "vector_truecount_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23539 ins_encode %{
23540 int opcode = this->ideal_Opcode();
23541 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23542 int mask_len = Matcher::vector_length(this, $mask);
23543 int vlen_enc = vector_length_encoding(this, $mask);
23544 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23545 $tmp$$Register, mask_len, mbt, vlen_enc);
23546 %}
23547 ins_pipe( pipe_slow );
23548 %}
23549
23550 instruct vmask_first_or_last_true_evex(rRegI dst, kReg mask, rRegL tmp, rFlagsReg cr) %{
23551 predicate(n->in(1)->bottom_type()->isa_pvectmask());
23552 match(Set dst (VectorMaskFirstTrue mask));
23553 match(Set dst (VectorMaskLastTrue mask));
23554 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
23555 format %{ "vector_mask_first_or_last_true_evex $dst, $mask \t! using $tmp as TEMP" %}
23556 ins_encode %{
23557 int opcode = this->ideal_Opcode();
23558 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23559 int mask_len = Matcher::vector_length(this, $mask);
23560 int mask_size = mask_len * type2aelembytes(mbt);
23561 int vlen_enc = vector_length_encoding(this, $mask);
23562 __ vector_mask_operation(opcode, $dst$$Register, $mask$$KRegister,
23563 $tmp$$Register, mask_len, mask_size, vlen_enc);
23564 %}
23565 ins_pipe( pipe_slow );
23566 %}
23567
23568 instruct vmask_first_or_last_true_bool(rRegI dst, vec mask, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23569 predicate(n->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23570 match(Set dst (VectorMaskFirstTrue mask));
23571 match(Set dst (VectorMaskLastTrue mask));
23572 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23573 format %{ "vector_mask_first_or_last_true_bool $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23574 ins_encode %{
23575 int opcode = this->ideal_Opcode();
23576 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23577 int mask_len = Matcher::vector_length(this, $mask);
23578 int vlen_enc = vector_length_encoding(this, $mask);
23579 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23580 $tmp$$Register, mask_len, mbt, vlen_enc);
23581 %}
23582 ins_pipe( pipe_slow );
23583 %}
23584
23585 instruct vmask_first_or_last_true_avx(rRegI dst, vec mask, immI size, rRegL tmp, vec xtmp, rFlagsReg cr) %{
23586 predicate(n->in(1)->in(1)->bottom_type()->isa_pvectmask() == nullptr);
23587 match(Set dst (VectorMaskFirstTrue (VectorStoreMask mask size)));
23588 match(Set dst (VectorMaskLastTrue (VectorStoreMask mask size)));
23589 effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, KILL cr);
23590 format %{ "vector_mask_first_or_last_true_avx $dst, $mask \t! using $tmp, $xtmp as TEMP" %}
23591 ins_encode %{
23592 int opcode = this->ideal_Opcode();
23593 BasicType mbt = Matcher::vector_element_basic_type(this, $mask);
23594 int mask_len = Matcher::vector_length(this, $mask);
23595 int vlen_enc = vector_length_encoding(this, $mask);
23596 __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister,
23597 $tmp$$Register, mask_len, mbt, vlen_enc);
23598 %}
23599 ins_pipe( pipe_slow );
23600 %}
23601
23602 // --------------------------------- Compress/Expand Operations ---------------------------
23603 instruct vcompress_reg_avx(vec dst, vec src, vec mask, rRegI rtmp, rRegL rscratch, vec perm, vec xtmp, rFlagsReg cr) %{
23604 predicate(!VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n) <= 32);
23605 match(Set dst (CompressV src mask));
23606 match(Set dst (ExpandV src mask));
23607 effect(TEMP_DEF dst, TEMP perm, TEMP xtmp, TEMP rtmp, TEMP rscratch, KILL cr);
23608 format %{ "vector_compress $dst, $src, $mask \t!using $xtmp, $rtmp, $rscratch and $perm as TEMP" %}
23609 ins_encode %{
23610 int opcode = this->ideal_Opcode();
23611 int vlen_enc = vector_length_encoding(this);
23612 BasicType bt = Matcher::vector_element_basic_type(this);
23613 __ vector_compress_expand_avx2(opcode, $dst$$XMMRegister, $src$$XMMRegister, $mask$$XMMRegister, $rtmp$$Register,
23614 $rscratch$$Register, $perm$$XMMRegister, $xtmp$$XMMRegister, bt, vlen_enc);
23615 %}
23616 ins_pipe( pipe_slow );
23617 %}
23618
23619 instruct vcompress_expand_reg_evex(vec dst, vec src, kReg mask) %{
23620 predicate(VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64);
23621 match(Set dst (CompressV src mask));
23622 match(Set dst (ExpandV src mask));
23623 format %{ "vector_compress_expand $dst, $src, $mask" %}
23624 ins_encode %{
23625 int opcode = this->ideal_Opcode();
23626 int vector_len = vector_length_encoding(this);
23627 BasicType bt = Matcher::vector_element_basic_type(this);
23628 __ vector_compress_expand(opcode, $dst$$XMMRegister, $src$$XMMRegister, $mask$$KRegister, false, bt, vector_len);
23629 %}
23630 ins_pipe( pipe_slow );
23631 %}
23632
23633 instruct vcompress_mask_reg_evex(kReg dst, kReg mask, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
23634 match(Set dst (CompressM mask));
23635 effect(TEMP rtmp1, TEMP rtmp2, KILL cr);
23636 format %{ "mask_compress_evex $dst, $mask\t! using $rtmp1 and $rtmp2 as TEMP" %}
23637 ins_encode %{
23638 assert(this->in(1)->bottom_type()->isa_pvectmask(), "");
23639 int mask_len = Matcher::vector_length(this);
23640 __ vector_mask_compress($dst$$KRegister, $mask$$KRegister, $rtmp1$$Register, $rtmp2$$Register, mask_len);
23641 %}
23642 ins_pipe( pipe_slow );
23643 %}
23644
23645 // -------------------------------- Bit and Byte Reversal Vector Operations ------------------------
23646
23647 instruct vreverse_reg(vec dst, vec src, vec xtmp1, vec xtmp2, rRegI rtmp) %{
23648 predicate(!VM_Version::supports_gfni());
23649 match(Set dst (ReverseV src));
23650 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp);
23651 format %{ "vector_reverse_bit_evex $dst, $src!\t using $xtmp1, $xtmp2 and $rtmp as TEMP" %}
23652 ins_encode %{
23653 int vec_enc = vector_length_encoding(this);
23654 BasicType bt = Matcher::vector_element_basic_type(this);
23655 __ vector_reverse_bit(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23656 $xtmp2$$XMMRegister, $rtmp$$Register, vec_enc);
23657 %}
23658 ins_pipe( pipe_slow );
23659 %}
23660
23661 instruct vreverse_reg_gfni(vec dst, vec src, vec xtmp) %{
23662 predicate(VM_Version::supports_gfni());
23663 match(Set dst (ReverseV src));
23664 effect(TEMP dst, TEMP xtmp);
23665 format %{ "vector_reverse_bit_gfni $dst, $src!\t using $xtmp as TEMP" %}
23666 ins_encode %{
23667 int vec_enc = vector_length_encoding(this);
23668 BasicType bt = Matcher::vector_element_basic_type(this);
23669 InternalAddress addr = $constantaddress(jlong(0x8040201008040201));
23670 __ vector_reverse_bit_gfni(bt, $dst$$XMMRegister, $src$$XMMRegister, addr, vec_enc,
23671 $xtmp$$XMMRegister);
23672 %}
23673 ins_pipe( pipe_slow );
23674 %}
23675
23676 instruct vreverse_byte_reg(vec dst, vec src) %{
23677 predicate(VM_Version::supports_avx512bw() || Matcher::vector_length_in_bytes(n) < 64);
23678 match(Set dst (ReverseBytesV src));
23679 effect(TEMP dst);
23680 format %{ "vector_reverse_byte $dst, $src" %}
23681 ins_encode %{
23682 int vec_enc = vector_length_encoding(this);
23683 BasicType bt = Matcher::vector_element_basic_type(this);
23684 __ vector_reverse_byte(bt, $dst$$XMMRegister, $src$$XMMRegister, vec_enc);
23685 %}
23686 ins_pipe( pipe_slow );
23687 %}
23688
23689 instruct vreverse_byte64_reg(vec dst, vec src, vec xtmp1, vec xtmp2, rRegI rtmp) %{
23690 predicate(!VM_Version::supports_avx512bw() && Matcher::vector_length_in_bytes(n) == 64);
23691 match(Set dst (ReverseBytesV src));
23692 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp);
23693 format %{ "vector_reverse_byte $dst, $src!\t using $xtmp1, $xtmp2 and $rtmp as TEMP" %}
23694 ins_encode %{
23695 int vec_enc = vector_length_encoding(this);
23696 BasicType bt = Matcher::vector_element_basic_type(this);
23697 __ vector_reverse_byte64(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23698 $xtmp2$$XMMRegister, $rtmp$$Register, vec_enc);
23699 %}
23700 ins_pipe( pipe_slow );
23701 %}
23702
23703 // ---------------------------------- Vector Count Leading Zeros -----------------------------------
23704
23705 instruct vcount_leading_zeros_IL_reg_evex(vec dst, vec src) %{
23706 predicate(is_clz_non_subword_predicate_evex(Matcher::vector_element_basic_type(n->in(1)),
23707 Matcher::vector_length_in_bytes(n->in(1))));
23708 match(Set dst (CountLeadingZerosV src));
23709 format %{ "vector_count_leading_zeros $dst, $src" %}
23710 ins_encode %{
23711 int vlen_enc = vector_length_encoding(this, $src);
23712 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23713 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, xnoreg,
23714 xnoreg, xnoreg, k0, noreg, true, vlen_enc);
23715 %}
23716 ins_pipe( pipe_slow );
23717 %}
23718
23719 instruct vcount_leading_zeros_IL_reg_evex_masked(vec dst, vec src, kReg mask) %{
23720 predicate(is_clz_non_subword_predicate_evex(Matcher::vector_element_basic_type(n->in(1)),
23721 Matcher::vector_length_in_bytes(n->in(1))));
23722 match(Set dst (CountLeadingZerosV src mask));
23723 format %{ "vector_count_leading_zeros $dst, $src, $mask" %}
23724 ins_encode %{
23725 int vlen_enc = vector_length_encoding(this, $src);
23726 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23727 __ evmovdquq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
23728 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, xnoreg, xnoreg,
23729 xnoreg, $mask$$KRegister, noreg, true, vlen_enc);
23730 %}
23731 ins_pipe( pipe_slow );
23732 %}
23733
23734 instruct vcount_leading_zeros_short_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2) %{
23735 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_SHORT &&
23736 VM_Version::supports_avx512cd() &&
23737 (VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64));
23738 match(Set dst (CountLeadingZerosV src));
23739 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
23740 format %{ "vector_count_leading_zeros $dst, $src!\t using $xtmp1 and $xtmp2 as TEMP" %}
23741 ins_encode %{
23742 int vlen_enc = vector_length_encoding(this, $src);
23743 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23744 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23745 $xtmp2$$XMMRegister, xnoreg, k0, noreg, true, vlen_enc);
23746 %}
23747 ins_pipe( pipe_slow );
23748 %}
23749
23750 instruct vcount_leading_zeros_byte_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, kReg ktmp, rRegP rtmp) %{
23751 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_BYTE && VM_Version::supports_avx512vlbw());
23752 match(Set dst (CountLeadingZerosV src));
23753 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP ktmp, TEMP rtmp);
23754 format %{ "vector_count_leading_zeros $dst, $src!\t using $xtmp1, $xtmp2, $xtmp3, $ktmp and $rtmp as TEMP" %}
23755 ins_encode %{
23756 int vlen_enc = vector_length_encoding(this, $src);
23757 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23758 __ vector_count_leading_zeros_evex(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23759 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $ktmp$$KRegister,
23760 $rtmp$$Register, true, vlen_enc);
23761 %}
23762 ins_pipe( pipe_slow );
23763 %}
23764
23765 instruct vcount_leading_zeros_int_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3) %{
23766 predicate(Matcher::vector_element_basic_type(n->in(1)) == T_INT &&
23767 !VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n->in(1)) < 64);
23768 match(Set dst (CountLeadingZerosV src));
23769 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3);
23770 format %{ "vector_count_leading_zeros $dst, $src\t! using $xtmp1, $xtmp2 and $xtmp3 as TEMP" %}
23771 ins_encode %{
23772 int vlen_enc = vector_length_encoding(this, $src);
23773 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23774 __ vector_count_leading_zeros_avx(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23775 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, noreg, vlen_enc);
23776 %}
23777 ins_pipe( pipe_slow );
23778 %}
23779
23780 instruct vcount_leading_zeros_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, rRegP rtmp) %{
23781 predicate(Matcher::vector_element_basic_type(n->in(1)) != T_INT &&
23782 !VM_Version::supports_avx512vl() && Matcher::vector_length_in_bytes(n->in(1)) < 64);
23783 match(Set dst (CountLeadingZerosV src));
23784 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP rtmp);
23785 format %{ "vector_count_leading_zeros $dst, $src\t! using $xtmp1, $xtmp2, $xtmp3, and $rtmp as TEMP" %}
23786 ins_encode %{
23787 int vlen_enc = vector_length_encoding(this, $src);
23788 BasicType bt = Matcher::vector_element_basic_type(this, $src);
23789 __ vector_count_leading_zeros_avx(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
23790 $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $rtmp$$Register, vlen_enc);
23791 %}
23792 ins_pipe( pipe_slow );
23793 %}
23794
23795 // ---------------------------------- Vector Masked Operations ------------------------------------
23796
23797 instruct vadd_reg_masked(vec dst, vec src2, kReg mask) %{
23798 match(Set dst (AddVB (Binary dst src2) mask));
23799 match(Set dst (AddVS (Binary dst src2) mask));
23800 match(Set dst (AddVI (Binary dst src2) mask));
23801 match(Set dst (AddVL (Binary dst src2) mask));
23802 match(Set dst (AddVF (Binary dst src2) mask));
23803 match(Set dst (AddVD (Binary dst src2) mask));
23804 format %{ "vpadd_masked $dst, $dst, $src2, $mask\t! add masked operation" %}
23805 ins_encode %{
23806 int vlen_enc = vector_length_encoding(this);
23807 BasicType bt = Matcher::vector_element_basic_type(this);
23808 int opc = this->ideal_Opcode();
23809 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23810 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
23811 %}
23812 ins_pipe( pipe_slow );
23813 %}
23814
23815 instruct vadd_mem_masked(vec dst, memory src2, kReg mask) %{
23816 match(Set dst (AddVB (Binary dst (LoadVector src2)) mask));
23817 match(Set dst (AddVS (Binary dst (LoadVector src2)) mask));
23818 match(Set dst (AddVI (Binary dst (LoadVector src2)) mask));
23819 match(Set dst (AddVL (Binary dst (LoadVector src2)) mask));
23820 match(Set dst (AddVF (Binary dst (LoadVector src2)) mask));
23821 match(Set dst (AddVD (Binary dst (LoadVector src2)) mask));
23822 format %{ "vpadd_masked $dst, $dst, $src2, $mask\t! add masked operation" %}
23823 ins_encode %{
23824 int vlen_enc = vector_length_encoding(this);
23825 BasicType bt = Matcher::vector_element_basic_type(this);
23826 int opc = this->ideal_Opcode();
23827 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23828 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
23829 %}
23830 ins_pipe( pipe_slow );
23831 %}
23832
23833 instruct vxor_reg_masked(vec dst, vec src2, kReg mask) %{
23834 match(Set dst (XorV (Binary dst src2) mask));
23835 format %{ "vxor_masked $dst, $dst, $src2, $mask\t! xor masked operation" %}
23836 ins_encode %{
23837 int vlen_enc = vector_length_encoding(this);
23838 BasicType bt = Matcher::vector_element_basic_type(this);
23839 int opc = this->ideal_Opcode();
23840 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23841 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
23842 %}
23843 ins_pipe( pipe_slow );
23844 %}
23845
23846 instruct vxor_mem_masked(vec dst, memory src2, kReg mask) %{
23847 match(Set dst (XorV (Binary dst (LoadVector src2)) mask));
23848 format %{ "vxor_masked $dst, $dst, $src2, $mask\t! xor masked operation" %}
23849 ins_encode %{
23850 int vlen_enc = vector_length_encoding(this);
23851 BasicType bt = Matcher::vector_element_basic_type(this);
23852 int opc = this->ideal_Opcode();
23853 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23854 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
23855 %}
23856 ins_pipe( pipe_slow );
23857 %}
23858
23859 instruct vor_reg_masked(vec dst, vec src2, kReg mask) %{
23860 match(Set dst (OrV (Binary dst src2) mask));
23861 format %{ "vor_masked $dst, $dst, $src2, $mask\t! or masked operation" %}
23862 ins_encode %{
23863 int vlen_enc = vector_length_encoding(this);
23864 BasicType bt = Matcher::vector_element_basic_type(this);
23865 int opc = this->ideal_Opcode();
23866 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23867 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
23868 %}
23869 ins_pipe( pipe_slow );
23870 %}
23871
23872 instruct vor_mem_masked(vec dst, memory src2, kReg mask) %{
23873 match(Set dst (OrV (Binary dst (LoadVector src2)) mask));
23874 format %{ "vor_masked $dst, $dst, $src2, $mask\t! or masked operation" %}
23875 ins_encode %{
23876 int vlen_enc = vector_length_encoding(this);
23877 BasicType bt = Matcher::vector_element_basic_type(this);
23878 int opc = this->ideal_Opcode();
23879 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23880 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
23881 %}
23882 ins_pipe( pipe_slow );
23883 %}
23884
23885 instruct vand_reg_masked(vec dst, vec src2, kReg mask) %{
23886 match(Set dst (AndV (Binary dst src2) mask));
23887 format %{ "vand_masked $dst, $dst, $src2, $mask\t! and masked operation" %}
23888 ins_encode %{
23889 int vlen_enc = vector_length_encoding(this);
23890 BasicType bt = Matcher::vector_element_basic_type(this);
23891 int opc = this->ideal_Opcode();
23892 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23893 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
23894 %}
23895 ins_pipe( pipe_slow );
23896 %}
23897
23898 instruct vand_mem_masked(vec dst, memory src2, kReg mask) %{
23899 match(Set dst (AndV (Binary dst (LoadVector src2)) mask));
23900 format %{ "vand_masked $dst, $dst, $src2, $mask\t! and masked operation" %}
23901 ins_encode %{
23902 int vlen_enc = vector_length_encoding(this);
23903 BasicType bt = Matcher::vector_element_basic_type(this);
23904 int opc = this->ideal_Opcode();
23905 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23906 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
23907 %}
23908 ins_pipe( pipe_slow );
23909 %}
23910
23911 instruct vsub_reg_masked(vec dst, vec src2, kReg mask) %{
23912 match(Set dst (SubVB (Binary dst src2) mask));
23913 match(Set dst (SubVS (Binary dst src2) mask));
23914 match(Set dst (SubVI (Binary dst src2) mask));
23915 match(Set dst (SubVL (Binary dst src2) mask));
23916 match(Set dst (SubVF (Binary dst src2) mask));
23917 match(Set dst (SubVD (Binary dst src2) mask));
23918 format %{ "vpsub_masked $dst, $dst, $src2, $mask\t! sub masked operation" %}
23919 ins_encode %{
23920 int vlen_enc = vector_length_encoding(this);
23921 BasicType bt = Matcher::vector_element_basic_type(this);
23922 int opc = this->ideal_Opcode();
23923 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23924 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
23925 %}
23926 ins_pipe( pipe_slow );
23927 %}
23928
23929 instruct vsub_mem_masked(vec dst, memory src2, kReg mask) %{
23930 match(Set dst (SubVB (Binary dst (LoadVector src2)) mask));
23931 match(Set dst (SubVS (Binary dst (LoadVector src2)) mask));
23932 match(Set dst (SubVI (Binary dst (LoadVector src2)) mask));
23933 match(Set dst (SubVL (Binary dst (LoadVector src2)) mask));
23934 match(Set dst (SubVF (Binary dst (LoadVector src2)) mask));
23935 match(Set dst (SubVD (Binary dst (LoadVector src2)) mask));
23936 format %{ "vpsub_masked $dst, $dst, $src2, $mask\t! sub masked operation" %}
23937 ins_encode %{
23938 int vlen_enc = vector_length_encoding(this);
23939 BasicType bt = Matcher::vector_element_basic_type(this);
23940 int opc = this->ideal_Opcode();
23941 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23942 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
23943 %}
23944 ins_pipe( pipe_slow );
23945 %}
23946
23947 instruct vmul_reg_masked(vec dst, vec src2, kReg mask) %{
23948 match(Set dst (MulVS (Binary dst src2) mask));
23949 match(Set dst (MulVI (Binary dst src2) mask));
23950 match(Set dst (MulVL (Binary dst src2) mask));
23951 match(Set dst (MulVF (Binary dst src2) mask));
23952 match(Set dst (MulVD (Binary dst src2) mask));
23953 format %{ "vpmul_masked $dst, $dst, $src2, $mask\t! mul masked operation" %}
23954 ins_encode %{
23955 int vlen_enc = vector_length_encoding(this);
23956 BasicType bt = Matcher::vector_element_basic_type(this);
23957 int opc = this->ideal_Opcode();
23958 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23959 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
23960 %}
23961 ins_pipe( pipe_slow );
23962 %}
23963
23964 instruct vmul_mem_masked(vec dst, memory src2, kReg mask) %{
23965 match(Set dst (MulVS (Binary dst (LoadVector src2)) mask));
23966 match(Set dst (MulVI (Binary dst (LoadVector src2)) mask));
23967 match(Set dst (MulVL (Binary dst (LoadVector src2)) mask));
23968 match(Set dst (MulVF (Binary dst (LoadVector src2)) mask));
23969 match(Set dst (MulVD (Binary dst (LoadVector src2)) mask));
23970 format %{ "vpmul_masked $dst, $dst, $src2, $mask\t! mul masked operation" %}
23971 ins_encode %{
23972 int vlen_enc = vector_length_encoding(this);
23973 BasicType bt = Matcher::vector_element_basic_type(this);
23974 int opc = this->ideal_Opcode();
23975 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23976 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
23977 %}
23978 ins_pipe( pipe_slow );
23979 %}
23980
23981 instruct vsqrt_reg_masked(vec dst, kReg mask) %{
23982 match(Set dst (SqrtVF dst mask));
23983 match(Set dst (SqrtVD dst mask));
23984 format %{ "vpsqrt_masked $dst, $mask\t! sqrt masked operation" %}
23985 ins_encode %{
23986 int vlen_enc = vector_length_encoding(this);
23987 BasicType bt = Matcher::vector_element_basic_type(this);
23988 int opc = this->ideal_Opcode();
23989 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
23990 $dst$$XMMRegister, $dst$$XMMRegister, true, vlen_enc);
23991 %}
23992 ins_pipe( pipe_slow );
23993 %}
23994
23995 instruct vdiv_reg_masked(vec dst, vec src2, kReg mask) %{
23996 match(Set dst (DivVF (Binary dst src2) mask));
23997 match(Set dst (DivVD (Binary dst src2) mask));
23998 format %{ "vpdiv_masked $dst, $dst, $src2, $mask\t! div masked operation" %}
23999 ins_encode %{
24000 int vlen_enc = vector_length_encoding(this);
24001 BasicType bt = Matcher::vector_element_basic_type(this);
24002 int opc = this->ideal_Opcode();
24003 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24004 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24005 %}
24006 ins_pipe( pipe_slow );
24007 %}
24008
24009 instruct vdiv_mem_masked(vec dst, memory src2, kReg mask) %{
24010 match(Set dst (DivVF (Binary dst (LoadVector src2)) mask));
24011 match(Set dst (DivVD (Binary dst (LoadVector src2)) mask));
24012 format %{ "vpdiv_masked $dst, $dst, $src2, $mask\t! div masked operation" %}
24013 ins_encode %{
24014 int vlen_enc = vector_length_encoding(this);
24015 BasicType bt = Matcher::vector_element_basic_type(this);
24016 int opc = this->ideal_Opcode();
24017 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24018 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24019 %}
24020 ins_pipe( pipe_slow );
24021 %}
24022
24023
24024 instruct vrol_imm_masked(vec dst, immI8 shift, kReg mask) %{
24025 match(Set dst (RotateLeftV (Binary dst shift) mask));
24026 match(Set dst (RotateRightV (Binary dst shift) mask));
24027 format %{ "vprotate_imm_masked $dst, $dst, $shift, $mask\t! rotate masked operation" %}
24028 ins_encode %{
24029 int vlen_enc = vector_length_encoding(this);
24030 BasicType bt = Matcher::vector_element_basic_type(this);
24031 int opc = this->ideal_Opcode();
24032 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24033 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24034 %}
24035 ins_pipe( pipe_slow );
24036 %}
24037
24038 instruct vrol_reg_masked(vec dst, vec src2, kReg mask) %{
24039 match(Set dst (RotateLeftV (Binary dst src2) mask));
24040 match(Set dst (RotateRightV (Binary dst src2) mask));
24041 format %{ "vrotate_masked $dst, $dst, $src2, $mask\t! rotate masked operation" %}
24042 ins_encode %{
24043 int vlen_enc = vector_length_encoding(this);
24044 BasicType bt = Matcher::vector_element_basic_type(this);
24045 int opc = this->ideal_Opcode();
24046 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24047 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24048 %}
24049 ins_pipe( pipe_slow );
24050 %}
24051
24052 instruct vlshift_imm_masked(vec dst, immI8 shift, kReg mask) %{
24053 match(Set dst (LShiftVS (Binary dst (LShiftCntV shift)) mask));
24054 match(Set dst (LShiftVI (Binary dst (LShiftCntV shift)) mask));
24055 match(Set dst (LShiftVL (Binary dst (LShiftCntV shift)) mask));
24056 format %{ "vplshift_imm_masked $dst, $dst, $shift, $mask\t! lshift masked operation" %}
24057 ins_encode %{
24058 int vlen_enc = vector_length_encoding(this);
24059 BasicType bt = Matcher::vector_element_basic_type(this);
24060 int opc = this->ideal_Opcode();
24061 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24062 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24063 %}
24064 ins_pipe( pipe_slow );
24065 %}
24066
24067 instruct vlshift_reg_masked(vec dst, vec src2, kReg mask) %{
24068 predicate(!n->as_ShiftV()->is_var_shift());
24069 match(Set dst (LShiftVS (Binary dst src2) mask));
24070 match(Set dst (LShiftVI (Binary dst src2) mask));
24071 match(Set dst (LShiftVL (Binary dst src2) mask));
24072 format %{ "vplshift_masked $dst, $dst, $src2, $mask\t! lshift masked operation" %}
24073 ins_encode %{
24074 int vlen_enc = vector_length_encoding(this);
24075 BasicType bt = Matcher::vector_element_basic_type(this);
24076 int opc = this->ideal_Opcode();
24077 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24078 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, false);
24079 %}
24080 ins_pipe( pipe_slow );
24081 %}
24082
24083 instruct vlshiftv_reg_masked(vec dst, vec src2, kReg mask) %{
24084 predicate(n->as_ShiftV()->is_var_shift());
24085 match(Set dst (LShiftVS (Binary dst src2) mask));
24086 match(Set dst (LShiftVI (Binary dst src2) mask));
24087 match(Set dst (LShiftVL (Binary dst src2) mask));
24088 format %{ "vplshiftv_masked $dst, $dst, $src2, $mask\t! lshift masked operation" %}
24089 ins_encode %{
24090 int vlen_enc = vector_length_encoding(this);
24091 BasicType bt = Matcher::vector_element_basic_type(this);
24092 int opc = this->ideal_Opcode();
24093 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24094 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, true);
24095 %}
24096 ins_pipe( pipe_slow );
24097 %}
24098
24099 instruct vrshift_imm_masked(vec dst, immI8 shift, kReg mask) %{
24100 match(Set dst (RShiftVS (Binary dst (RShiftCntV shift)) mask));
24101 match(Set dst (RShiftVI (Binary dst (RShiftCntV shift)) mask));
24102 match(Set dst (RShiftVL (Binary dst (RShiftCntV shift)) mask));
24103 format %{ "vprshift_imm_masked $dst, $dst, $shift, $mask\t! rshift masked operation" %}
24104 ins_encode %{
24105 int vlen_enc = vector_length_encoding(this);
24106 BasicType bt = Matcher::vector_element_basic_type(this);
24107 int opc = this->ideal_Opcode();
24108 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24109 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24110 %}
24111 ins_pipe( pipe_slow );
24112 %}
24113
24114 instruct vrshift_reg_masked(vec dst, vec src2, kReg mask) %{
24115 predicate(!n->as_ShiftV()->is_var_shift());
24116 match(Set dst (RShiftVS (Binary dst src2) mask));
24117 match(Set dst (RShiftVI (Binary dst src2) mask));
24118 match(Set dst (RShiftVL (Binary dst src2) mask));
24119 format %{ "vprshift_masked $dst, $dst, $src2, $mask\t! rshift masked operation" %}
24120 ins_encode %{
24121 int vlen_enc = vector_length_encoding(this);
24122 BasicType bt = Matcher::vector_element_basic_type(this);
24123 int opc = this->ideal_Opcode();
24124 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24125 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, false);
24126 %}
24127 ins_pipe( pipe_slow );
24128 %}
24129
24130 instruct vrshiftv_reg_masked(vec dst, vec src2, kReg mask) %{
24131 predicate(n->as_ShiftV()->is_var_shift());
24132 match(Set dst (RShiftVS (Binary dst src2) mask));
24133 match(Set dst (RShiftVI (Binary dst src2) mask));
24134 match(Set dst (RShiftVL (Binary dst src2) mask));
24135 format %{ "vprshiftv_masked $dst, $dst, $src2, $mask\t! rshift masked operation" %}
24136 ins_encode %{
24137 int vlen_enc = vector_length_encoding(this);
24138 BasicType bt = Matcher::vector_element_basic_type(this);
24139 int opc = this->ideal_Opcode();
24140 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24141 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, true);
24142 %}
24143 ins_pipe( pipe_slow );
24144 %}
24145
24146 instruct vurshift_imm_masked(vec dst, immI8 shift, kReg mask) %{
24147 match(Set dst (URShiftVS (Binary dst (RShiftCntV shift)) mask));
24148 match(Set dst (URShiftVI (Binary dst (RShiftCntV shift)) mask));
24149 match(Set dst (URShiftVL (Binary dst (RShiftCntV shift)) mask));
24150 format %{ "vpurshift_imm_masked $dst, $dst, $shift, $mask\t! urshift masked operation" %}
24151 ins_encode %{
24152 int vlen_enc = vector_length_encoding(this);
24153 BasicType bt = Matcher::vector_element_basic_type(this);
24154 int opc = this->ideal_Opcode();
24155 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24156 $dst$$XMMRegister, $shift$$constant, true, vlen_enc);
24157 %}
24158 ins_pipe( pipe_slow );
24159 %}
24160
24161 instruct vurshift_reg_masked(vec dst, vec src2, kReg mask) %{
24162 predicate(!n->as_ShiftV()->is_var_shift());
24163 match(Set dst (URShiftVS (Binary dst src2) mask));
24164 match(Set dst (URShiftVI (Binary dst src2) mask));
24165 match(Set dst (URShiftVL (Binary dst src2) mask));
24166 format %{ "vpurshift_masked $dst, $dst, $src2, $mask\t! urshift masked operation" %}
24167 ins_encode %{
24168 int vlen_enc = vector_length_encoding(this);
24169 BasicType bt = Matcher::vector_element_basic_type(this);
24170 int opc = this->ideal_Opcode();
24171 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24172 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, false);
24173 %}
24174 ins_pipe( pipe_slow );
24175 %}
24176
24177 instruct vurshiftv_reg_masked(vec dst, vec src2, kReg mask) %{
24178 predicate(n->as_ShiftV()->is_var_shift());
24179 match(Set dst (URShiftVS (Binary dst src2) mask));
24180 match(Set dst (URShiftVI (Binary dst src2) mask));
24181 match(Set dst (URShiftVL (Binary dst src2) mask));
24182 format %{ "vpurshiftv_masked $dst, $dst, $src2, $mask\t! urshift masked operation" %}
24183 ins_encode %{
24184 int vlen_enc = vector_length_encoding(this);
24185 BasicType bt = Matcher::vector_element_basic_type(this);
24186 int opc = this->ideal_Opcode();
24187 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24188 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc, true);
24189 %}
24190 ins_pipe( pipe_slow );
24191 %}
24192
24193 instruct vmaxv_reg_masked(vec dst, vec src2, kReg mask) %{
24194 match(Set dst (MaxV (Binary dst src2) mask));
24195 format %{ "vpmax_masked $dst, $dst, $src2, $mask\t! max masked operation" %}
24196 ins_encode %{
24197 int vlen_enc = vector_length_encoding(this);
24198 BasicType bt = Matcher::vector_element_basic_type(this);
24199 int opc = this->ideal_Opcode();
24200 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24201 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24202 %}
24203 ins_pipe( pipe_slow );
24204 %}
24205
24206 instruct vmaxv_mem_masked(vec dst, memory src2, kReg mask) %{
24207 match(Set dst (MaxV (Binary dst (LoadVector src2)) mask));
24208 format %{ "vpmax_masked $dst, $dst, $src2, $mask\t! max masked operation" %}
24209 ins_encode %{
24210 int vlen_enc = vector_length_encoding(this);
24211 BasicType bt = Matcher::vector_element_basic_type(this);
24212 int opc = this->ideal_Opcode();
24213 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24214 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24215 %}
24216 ins_pipe( pipe_slow );
24217 %}
24218
24219 instruct vminv_reg_masked(vec dst, vec src2, kReg mask) %{
24220 match(Set dst (MinV (Binary dst src2) mask));
24221 format %{ "vpmin_masked $dst, $dst, $src2, $mask\t! min masked operation" %}
24222 ins_encode %{
24223 int vlen_enc = vector_length_encoding(this);
24224 BasicType bt = Matcher::vector_element_basic_type(this);
24225 int opc = this->ideal_Opcode();
24226 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24227 $dst$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24228 %}
24229 ins_pipe( pipe_slow );
24230 %}
24231
24232 instruct vminv_mem_masked(vec dst, memory src2, kReg mask) %{
24233 match(Set dst (MinV (Binary dst (LoadVector src2)) mask));
24234 format %{ "vpmin_masked $dst, $dst, $src2, $mask\t! min masked operation" %}
24235 ins_encode %{
24236 int vlen_enc = vector_length_encoding(this);
24237 BasicType bt = Matcher::vector_element_basic_type(this);
24238 int opc = this->ideal_Opcode();
24239 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24240 $dst$$XMMRegister, $src2$$Address, true, vlen_enc);
24241 %}
24242 ins_pipe( pipe_slow );
24243 %}
24244
24245 instruct vrearrangev_reg_masked(vec dst, vec src2, kReg mask) %{
24246 match(Set dst (VectorRearrange (Binary dst src2) mask));
24247 format %{ "vprearrange_masked $dst, $dst, $src2, $mask\t! rearrange masked operation" %}
24248 ins_encode %{
24249 int vlen_enc = vector_length_encoding(this);
24250 BasicType bt = Matcher::vector_element_basic_type(this);
24251 int opc = this->ideal_Opcode();
24252 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24253 $dst$$XMMRegister, $src2$$XMMRegister, false, vlen_enc);
24254 %}
24255 ins_pipe( pipe_slow );
24256 %}
24257
24258 instruct vabs_masked(vec dst, kReg mask) %{
24259 match(Set dst (AbsVB dst mask));
24260 match(Set dst (AbsVS dst mask));
24261 match(Set dst (AbsVI dst mask));
24262 match(Set dst (AbsVL dst mask));
24263 format %{ "vabs_masked $dst, $mask \t! vabs masked operation" %}
24264 ins_encode %{
24265 int vlen_enc = vector_length_encoding(this);
24266 BasicType bt = Matcher::vector_element_basic_type(this);
24267 int opc = this->ideal_Opcode();
24268 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24269 $dst$$XMMRegister, $dst$$XMMRegister, true, vlen_enc);
24270 %}
24271 ins_pipe( pipe_slow );
24272 %}
24273
24274 instruct vfma_reg_masked(vec dst, vec src2, vec src3, kReg mask) %{
24275 match(Set dst (FmaVF (Binary dst src2) (Binary src3 mask)));
24276 match(Set dst (FmaVD (Binary dst src2) (Binary src3 mask)));
24277 format %{ "vfma_masked $dst, $src2, $src3, $mask \t! vfma masked operation" %}
24278 ins_encode %{
24279 assert(UseFMA, "Needs FMA instructions support.");
24280 int vlen_enc = vector_length_encoding(this);
24281 BasicType bt = Matcher::vector_element_basic_type(this);
24282 int opc = this->ideal_Opcode();
24283 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24284 $src2$$XMMRegister, $src3$$XMMRegister, true, vlen_enc);
24285 %}
24286 ins_pipe( pipe_slow );
24287 %}
24288
24289 instruct vfma_mem_masked(vec dst, vec src2, memory src3, kReg mask) %{
24290 match(Set dst (FmaVF (Binary dst src2) (Binary (LoadVector src3) mask)));
24291 match(Set dst (FmaVD (Binary dst src2) (Binary (LoadVector src3) mask)));
24292 format %{ "vfma_masked $dst, $src2, $src3, $mask \t! vfma masked operation" %}
24293 ins_encode %{
24294 assert(UseFMA, "Needs FMA instructions support.");
24295 int vlen_enc = vector_length_encoding(this);
24296 BasicType bt = Matcher::vector_element_basic_type(this);
24297 int opc = this->ideal_Opcode();
24298 __ evmasked_op(opc, bt, $mask$$KRegister, $dst$$XMMRegister,
24299 $src2$$XMMRegister, $src3$$Address, true, vlen_enc);
24300 %}
24301 ins_pipe( pipe_slow );
24302 %}
24303
24304 instruct evcmp_masked(kReg dst, vec src1, vec src2, immI8 cond, kReg mask) %{
24305 match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond mask)));
24306 format %{ "vcmp_masked $dst, $src1, $src2, $cond, $mask" %}
24307 ins_encode %{
24308 assert(bottom_type()->isa_pvectmask(), "TypePVectMask expected");
24309 int vlen_enc = vector_length_encoding(this, $src1);
24310 BasicType src1_elem_bt = Matcher::vector_element_basic_type(this, $src1);
24311
24312 // Comparison i
24313 switch (src1_elem_bt) {
24314 case T_BYTE: {
24315 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24316 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24317 __ evpcmpb($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24318 break;
24319 }
24320 case T_SHORT: {
24321 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24322 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24323 __ evpcmpw($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24324 break;
24325 }
24326 case T_INT: {
24327 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24328 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24329 __ evpcmpd($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24330 break;
24331 }
24332 case T_LONG: {
24333 bool is_unsigned = Matcher::is_unsigned_booltest_pred($cond$$constant);
24334 Assembler::ComparisonPredicate cmp = booltest_pred_to_comparison_pred($cond$$constant);
24335 __ evpcmpq($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, !is_unsigned, vlen_enc);
24336 break;
24337 }
24338 case T_FLOAT: {
24339 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
24340 __ evcmpps($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
24341 break;
24342 }
24343 case T_DOUBLE: {
24344 Assembler::ComparisonPredicateFP cmp = booltest_pred_to_comparison_pred_fp($cond$$constant);
24345 __ evcmppd($dst$$KRegister, $mask$$KRegister, $src1$$XMMRegister, $src2$$XMMRegister, cmp, vlen_enc);
24346 break;
24347 }
24348 default: assert(false, "%s", type2name(src1_elem_bt)); break;
24349 }
24350 %}
24351 ins_pipe( pipe_slow );
24352 %}
24353
24354 instruct mask_all_evexI_LE32(kReg dst, rRegI src) %{
24355 predicate(Matcher::vector_length(n) <= 32);
24356 match(Set dst (MaskAll src));
24357 format %{ "mask_all_evexI_LE32 $dst, $src \t" %}
24358 ins_encode %{
24359 int mask_len = Matcher::vector_length(this);
24360 __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
24361 %}
24362 ins_pipe( pipe_slow );
24363 %}
24364
24365 instruct mask_not_immLT8(kReg dst, kReg src, rRegI rtmp, kReg ktmp, immI_M1 cnt) %{
24366 predicate(Matcher::vector_length(n) < 8 && VM_Version::supports_avx512dq());
24367 match(Set dst (XorVMask src (MaskAll cnt)));
24368 effect(TEMP_DEF dst, TEMP rtmp, TEMP ktmp);
24369 format %{ "mask_not_LT8 $dst, $src, $cnt \t!using $ktmp and $rtmp as TEMP" %}
24370 ins_encode %{
24371 uint masklen = Matcher::vector_length(this);
24372 __ knot(masklen, $dst$$KRegister, $src$$KRegister, $ktmp$$KRegister, $rtmp$$Register);
24373 %}
24374 ins_pipe( pipe_slow );
24375 %}
24376
24377 instruct mask_not_imm(kReg dst, kReg src, immI_M1 cnt) %{
24378 predicate((Matcher::vector_length(n) == 8 && VM_Version::supports_avx512dq()) ||
24379 (Matcher::vector_length(n) == 16) ||
24380 (Matcher::vector_length(n) > 16 && VM_Version::supports_avx512bw()));
24381 match(Set dst (XorVMask src (MaskAll cnt)));
24382 format %{ "mask_not $dst, $src, $cnt \t! mask not operation" %}
24383 ins_encode %{
24384 uint masklen = Matcher::vector_length(this);
24385 __ knot(masklen, $dst$$KRegister, $src$$KRegister);
24386 %}
24387 ins_pipe( pipe_slow );
24388 %}
24389
24390 instruct long_to_maskLE8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2) %{
24391 predicate(n->bottom_type()->isa_pvectmask() == nullptr && Matcher::vector_length(n) <= 8);
24392 match(Set dst (VectorLongToMask src));
24393 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2);
24394 format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2" %}
24395 ins_encode %{
24396 int mask_len = Matcher::vector_length(this);
24397 int vec_enc = vector_length_encoding(mask_len);
24398 __ vector_long_to_maskvec($dst$$XMMRegister, $src$$Register, $rtmp1$$Register,
24399 $rtmp2$$Register, xnoreg, mask_len, vec_enc);
24400 %}
24401 ins_pipe( pipe_slow );
24402 %}
24403
24404
24405 instruct long_to_maskGT8_avx(vec dst, rRegL src, rRegL rtmp1, rRegL rtmp2, vec xtmp1, rFlagsReg cr) %{
24406 predicate(n->bottom_type()->isa_pvectmask() == nullptr && Matcher::vector_length(n) > 8);
24407 match(Set dst (VectorLongToMask src));
24408 effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, TEMP xtmp1, KILL cr);
24409 format %{ "long_to_mask_avx $dst, $src\t! using $rtmp1, $rtmp2, $xtmp1, as TEMP" %}
24410 ins_encode %{
24411 int mask_len = Matcher::vector_length(this);
24412 assert(mask_len <= 32, "invalid mask length");
24413 int vec_enc = vector_length_encoding(mask_len);
24414 __ vector_long_to_maskvec($dst$$XMMRegister, $src$$Register, $rtmp1$$Register,
24415 $rtmp2$$Register, $xtmp1$$XMMRegister, mask_len, vec_enc);
24416 %}
24417 ins_pipe( pipe_slow );
24418 %}
24419
24420 instruct long_to_mask_evex(kReg dst, rRegL src) %{
24421 predicate(n->bottom_type()->isa_pvectmask());
24422 match(Set dst (VectorLongToMask src));
24423 format %{ "long_to_mask_evex $dst, $src\t!" %}
24424 ins_encode %{
24425 __ kmov($dst$$KRegister, $src$$Register);
24426 %}
24427 ins_pipe( pipe_slow );
24428 %}
24429
24430 instruct mask_opers_evex(kReg dst, kReg src1, kReg src2, kReg kscratch) %{
24431 match(Set dst (AndVMask src1 src2));
24432 match(Set dst (OrVMask src1 src2));
24433 match(Set dst (XorVMask src1 src2));
24434 effect(TEMP kscratch);
24435 format %{ "mask_opers_evex $dst, $src1, $src2\t! using $kscratch as TEMP" %}
24436 ins_encode %{
24437 const MachNode* mask1 = static_cast<const MachNode*>(this->in(this->operand_index($src1)));
24438 const MachNode* mask2 = static_cast<const MachNode*>(this->in(this->operand_index($src2)));
24439 assert(Type::equals(mask1->bottom_type(), mask2->bottom_type()), "Mask types must be equal");
24440 uint masklen = Matcher::vector_length(this);
24441 masklen = (masklen < 16 && !VM_Version::supports_avx512dq()) ? 16 : masklen;
24442 __ masked_op(this->ideal_Opcode(), masklen, $dst$$KRegister, $src1$$KRegister, $src2$$KRegister);
24443 %}
24444 ins_pipe( pipe_slow );
24445 %}
24446
24447 instruct vternlog_reg_masked(vec dst, vec src2, vec src3, immU8 func, kReg mask) %{
24448 match(Set dst (MacroLogicV dst (Binary src2 (Binary src3 (Binary func mask)))));
24449 format %{ "vternlog_masked $dst,$src2,$src3,$func,$mask\t! vternlog masked operation" %}
24450 ins_encode %{
24451 int vlen_enc = vector_length_encoding(this);
24452 BasicType bt = Matcher::vector_element_basic_type(this);
24453 __ evpternlog($dst$$XMMRegister, $func$$constant, $mask$$KRegister,
24454 $src2$$XMMRegister, $src3$$XMMRegister, true, bt, vlen_enc);
24455 %}
24456 ins_pipe( pipe_slow );
24457 %}
24458
24459 instruct vternlogd_mem_masked(vec dst, vec src2, memory src3, immU8 func, kReg mask) %{
24460 match(Set dst (MacroLogicV dst (Binary src2 (Binary src3 (Binary func mask)))));
24461 format %{ "vternlog_masked $dst,$src2,$src3,$func,$mask\t! vternlog masked operation" %}
24462 ins_encode %{
24463 int vlen_enc = vector_length_encoding(this);
24464 BasicType bt = Matcher::vector_element_basic_type(this);
24465 __ evpternlog($dst$$XMMRegister, $func$$constant, $mask$$KRegister,
24466 $src2$$XMMRegister, $src3$$Address, true, bt, vlen_enc);
24467 %}
24468 ins_pipe( pipe_slow );
24469 %}
24470
24471 instruct castMM(kReg dst)
24472 %{
24473 match(Set dst (CastVV dst));
24474
24475 size(0);
24476 format %{ "# castVV of $dst" %}
24477 ins_encode(/* empty encoding */);
24478 ins_cost(0);
24479 ins_pipe(empty);
24480 %}
24481
24482 instruct castVV(vec dst)
24483 %{
24484 match(Set dst (CastVV dst));
24485
24486 size(0);
24487 format %{ "# castVV of $dst" %}
24488 ins_encode(/* empty encoding */);
24489 ins_cost(0);
24490 ins_pipe(empty);
24491 %}
24492
24493 instruct castVVLeg(legVec dst)
24494 %{
24495 match(Set dst (CastVV dst));
24496
24497 size(0);
24498 format %{ "# castVV of $dst" %}
24499 ins_encode(/* empty encoding */);
24500 ins_cost(0);
24501 ins_pipe(empty);
24502 %}
24503
24504 instruct FloatClassCheck_reg_reg_vfpclass(rRegI dst, regF src, kReg ktmp, rFlagsReg cr)
24505 %{
24506 match(Set dst (IsInfiniteF src));
24507 effect(TEMP ktmp, KILL cr);
24508 format %{ "float_class_check $dst, $src" %}
24509 ins_encode %{
24510 __ vfpclassss($ktmp$$KRegister, $src$$XMMRegister, 0x18);
24511 __ kmovbl($dst$$Register, $ktmp$$KRegister);
24512 %}
24513 ins_pipe(pipe_slow);
24514 %}
24515
24516 instruct DoubleClassCheck_reg_reg_vfpclass(rRegI dst, regD src, kReg ktmp, rFlagsReg cr)
24517 %{
24518 match(Set dst (IsInfiniteD src));
24519 effect(TEMP ktmp, KILL cr);
24520 format %{ "double_class_check $dst, $src" %}
24521 ins_encode %{
24522 __ vfpclasssd($ktmp$$KRegister, $src$$XMMRegister, 0x18);
24523 __ kmovbl($dst$$Register, $ktmp$$KRegister);
24524 %}
24525 ins_pipe(pipe_slow);
24526 %}
24527
24528 instruct vector_addsub_saturating_subword_reg(vec dst, vec src1, vec src2)
24529 %{
24530 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24531 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
24532 match(Set dst (SaturatingAddV src1 src2));
24533 match(Set dst (SaturatingSubV src1 src2));
24534 format %{ "vector_addsub_saturating_subword $dst, $src1, $src2" %}
24535 ins_encode %{
24536 int vlen_enc = vector_length_encoding(this);
24537 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24538 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24539 $src1$$XMMRegister, $src2$$XMMRegister, false, vlen_enc);
24540 %}
24541 ins_pipe(pipe_slow);
24542 %}
24543
24544 instruct vector_addsub_saturating_unsigned_subword_reg(vec dst, vec src1, vec src2)
24545 %{
24546 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24547 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
24548 match(Set dst (SaturatingAddV src1 src2));
24549 match(Set dst (SaturatingSubV src1 src2));
24550 format %{ "vector_addsub_saturating_unsigned_subword $dst, $src1, $src2" %}
24551 ins_encode %{
24552 int vlen_enc = vector_length_encoding(this);
24553 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24554 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24555 $src1$$XMMRegister, $src2$$XMMRegister, true, vlen_enc);
24556 %}
24557 ins_pipe(pipe_slow);
24558 %}
24559
24560 instruct vector_addsub_saturating_reg_evex(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2)
24561 %{
24562 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24563 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned() &&
24564 (Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()));
24565 match(Set dst (SaturatingAddV src1 src2));
24566 match(Set dst (SaturatingSubV src1 src2));
24567 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2);
24568 format %{ "vector_addsub_saturating_evex $dst, $src1, $src2 \t! using $xtmp1, $xtmp2, $ktmp1 and $ktmp2 as TEMP" %}
24569 ins_encode %{
24570 int vlen_enc = vector_length_encoding(this);
24571 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24572 __ vector_addsub_dq_saturating_evex(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24573 $src1$$XMMRegister, $src2$$XMMRegister,
24574 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister,
24575 $ktmp1$$KRegister, $ktmp2$$KRegister, vlen_enc);
24576 %}
24577 ins_pipe(pipe_slow);
24578 %}
24579
24580 instruct vector_addsub_saturating_reg_avx(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4)
24581 %{
24582 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24583 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned() &&
24584 Matcher::vector_length_in_bytes(n) <= 32 && !VM_Version::supports_avx512vl());
24585 match(Set dst (SaturatingAddV src1 src2));
24586 match(Set dst (SaturatingSubV src1 src2));
24587 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4);
24588 format %{ "vector_addsub_saturating_avx $dst, $src1, $src2 \t! using $xtmp1, $xtmp2, $xtmp3 and $xtmp4 as TEMP" %}
24589 ins_encode %{
24590 int vlen_enc = vector_length_encoding(this);
24591 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24592 __ vector_addsub_dq_saturating_avx(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister, $src1$$XMMRegister,
24593 $src2$$XMMRegister, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister,
24594 $xtmp3$$XMMRegister, $xtmp4$$XMMRegister, vlen_enc);
24595 %}
24596 ins_pipe(pipe_slow);
24597 %}
24598
24599 instruct vector_add_saturating_unsigned_reg_evex(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, kReg ktmp)
24600 %{
24601 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24602 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24603 (Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()));
24604 match(Set dst (SaturatingAddV src1 src2));
24605 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp);
24606 format %{ "vector_add_saturating_unsigned_evex $dst, $src1, $src2 \t! using $xtmp1, $xtmp2 and $ktmp as TEMP" %}
24607 ins_encode %{
24608 int vlen_enc = vector_length_encoding(this);
24609 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24610 __ vector_add_dq_saturating_unsigned_evex(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
24611 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp$$KRegister, vlen_enc);
24612 %}
24613 ins_pipe(pipe_slow);
24614 %}
24615
24616 instruct vector_add_saturating_unsigned_reg_avx(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2, vec xtmp3)
24617 %{
24618 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24619 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24620 Matcher::vector_length_in_bytes(n) <= 32 && !VM_Version::supports_avx512vl());
24621 match(Set dst (SaturatingAddV src1 src2));
24622 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3);
24623 format %{ "vector_add_saturating_unsigned_avx $dst, $src1, $src2 \t! using $xtmp1, $xtmp2 and $xtmp3 as TEMP" %}
24624 ins_encode %{
24625 int vlen_enc = vector_length_encoding(this);
24626 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24627 __ vector_add_dq_saturating_unsigned_avx(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
24628 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, vlen_enc);
24629 %}
24630 ins_pipe(pipe_slow);
24631 %}
24632
24633 instruct vector_sub_saturating_unsigned_reg_evex(vec dst, vec src1, vec src2, kReg ktmp)
24634 %{
24635 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24636 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24637 (Matcher::vector_length_in_bytes(n) == 64 || VM_Version::supports_avx512vl()));
24638 match(Set dst (SaturatingSubV src1 src2));
24639 effect(TEMP ktmp);
24640 format %{ "vector_sub_saturating_unsigned_evex $dst, $src1, $src2 \t! using $ktmp as TEMP" %}
24641 ins_encode %{
24642 int vlen_enc = vector_length_encoding(this);
24643 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24644 __ vector_sub_dq_saturating_unsigned_evex(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister,
24645 $src2$$XMMRegister, $ktmp$$KRegister, vlen_enc);
24646 %}
24647 ins_pipe(pipe_slow);
24648 %}
24649
24650 instruct vector_sub_saturating_unsigned_reg_avx(vec dst, vec src1, vec src2, vec xtmp1, vec xtmp2)
24651 %{
24652 predicate(!is_subword_type(Matcher::vector_element_basic_type(n)) &&
24653 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned() &&
24654 Matcher::vector_length_in_bytes(n) <= 32 && !VM_Version::supports_avx512vl());
24655 match(Set dst (SaturatingSubV src1 src2));
24656 effect(TEMP dst, TEMP xtmp1, TEMP xtmp2);
24657 format %{ "vector_sub_saturating_unsigned_avx $dst, $src1, $src2 \t! using $xtmp1 and $xtmp2 as TEMP" %}
24658 ins_encode %{
24659 int vlen_enc = vector_length_encoding(this);
24660 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24661 __ vector_sub_dq_saturating_unsigned_avx(elem_bt, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
24662 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
24663 %}
24664 ins_pipe(pipe_slow);
24665 %}
24666
24667 instruct vector_addsub_saturating_subword_mem(vec dst, vec src1, memory src2)
24668 %{
24669 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24670 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
24671 match(Set dst (SaturatingAddV src1 (LoadVector src2)));
24672 match(Set dst (SaturatingSubV src1 (LoadVector src2)));
24673 format %{ "vector_addsub_saturating_subword $dst, $src1, $src2" %}
24674 ins_encode %{
24675 int vlen_enc = vector_length_encoding(this);
24676 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24677 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24678 $src1$$XMMRegister, $src2$$Address, false, vlen_enc);
24679 %}
24680 ins_pipe(pipe_slow);
24681 %}
24682
24683 instruct vector_addsub_saturating_unsigned_subword_mem(vec dst, vec src1, memory src2)
24684 %{
24685 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24686 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
24687 match(Set dst (SaturatingAddV src1 (LoadVector src2)));
24688 match(Set dst (SaturatingSubV src1 (LoadVector src2)));
24689 format %{ "vector_addsub_saturating_unsigned_subword $dst, $src1, $src2" %}
24690 ins_encode %{
24691 int vlen_enc = vector_length_encoding(this);
24692 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24693 __ vector_saturating_op(this->ideal_Opcode(), elem_bt, $dst$$XMMRegister,
24694 $src1$$XMMRegister, $src2$$Address, true, vlen_enc);
24695 %}
24696 ins_pipe(pipe_slow);
24697 %}
24698
24699 instruct vector_addsub_saturating_subword_masked_reg(vec dst, vec src, kReg mask) %{
24700 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24701 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
24702 match(Set dst (SaturatingAddV (Binary dst src) mask));
24703 match(Set dst (SaturatingSubV (Binary dst src) mask));
24704 format %{ "vector_addsub_saturating_subword_masked $dst, $mask, $src" %}
24705 ins_encode %{
24706 int vlen_enc = vector_length_encoding(this);
24707 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24708 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
24709 $dst$$XMMRegister, $src$$XMMRegister, false, true, vlen_enc);
24710 %}
24711 ins_pipe( pipe_slow );
24712 %}
24713
24714 instruct vector_addsub_saturating_unsigned_subword_masked_reg(vec dst, vec src, kReg mask) %{
24715 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24716 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
24717 match(Set dst (SaturatingAddV (Binary dst src) mask));
24718 match(Set dst (SaturatingSubV (Binary dst src) mask));
24719 format %{ "vector_addsub_saturating_unsigned_subword_masked $dst, $mask, $src" %}
24720 ins_encode %{
24721 int vlen_enc = vector_length_encoding(this);
24722 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24723 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
24724 $dst$$XMMRegister, $src$$XMMRegister, true, true, vlen_enc);
24725 %}
24726 ins_pipe( pipe_slow );
24727 %}
24728
24729 instruct vector_addsub_saturating_subword_masked_mem(vec dst, memory src, kReg mask) %{
24730 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24731 n->is_SaturatingVector() && !n->as_SaturatingVector()->is_unsigned());
24732 match(Set dst (SaturatingAddV (Binary dst (LoadVector src)) mask));
24733 match(Set dst (SaturatingSubV (Binary dst (LoadVector src)) mask));
24734 format %{ "vector_addsub_saturating_subword_masked $dst, $mask, $src" %}
24735 ins_encode %{
24736 int vlen_enc = vector_length_encoding(this);
24737 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24738 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
24739 $dst$$XMMRegister, $src$$Address, false, true, vlen_enc);
24740 %}
24741 ins_pipe( pipe_slow );
24742 %}
24743
24744 instruct vector_addsub_saturating_unsigned_subword_masked_mem(vec dst, memory src, kReg mask) %{
24745 predicate(is_subword_type(Matcher::vector_element_basic_type(n)) &&
24746 n->is_SaturatingVector() && n->as_SaturatingVector()->is_unsigned());
24747 match(Set dst (SaturatingAddV (Binary dst (LoadVector src)) mask));
24748 match(Set dst (SaturatingSubV (Binary dst (LoadVector src)) mask));
24749 format %{ "vector_addsub_saturating_unsigned_subword_masked $dst, $mask, $src" %}
24750 ins_encode %{
24751 int vlen_enc = vector_length_encoding(this);
24752 BasicType elem_bt = Matcher::vector_element_basic_type(this);
24753 __ evmasked_saturating_op(this->ideal_Opcode(), elem_bt, $mask$$KRegister, $dst$$XMMRegister,
24754 $dst$$XMMRegister, $src$$Address, true, true, vlen_enc);
24755 %}
24756 ins_pipe( pipe_slow );
24757 %}
24758
24759 instruct vector_selectfrom_twovectors_reg_evex(vec index, vec src1, vec src2)
24760 %{
24761 match(Set index (SelectFromTwoVector (Binary index src1) src2));
24762 format %{ "select_from_two_vector $index, $src1, $src2 \t!" %}
24763 ins_encode %{
24764 int vlen_enc = vector_length_encoding(this);
24765 BasicType bt = Matcher::vector_element_basic_type(this);
24766 __ select_from_two_vectors_evex(bt, $index$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
24767 %}
24768 ins_pipe(pipe_slow);
24769 %}
24770
24771 instruct reinterpretS2HF(regF dst, rRegI src)
24772 %{
24773 match(Set dst (ReinterpretS2HF src));
24774 format %{ "evmovw $dst, $src" %}
24775 ins_encode %{
24776 __ evmovw($dst$$XMMRegister, $src$$Register);
24777 %}
24778 ins_pipe(pipe_slow);
24779 %}
24780
24781 instruct reinterpretHF2S(rRegI dst, regF src)
24782 %{
24783 match(Set dst (ReinterpretHF2S src));
24784 format %{ "evmovw $dst, $src" %}
24785 ins_encode %{
24786 __ evmovw($dst$$Register, $src$$XMMRegister);
24787 __ narrow_subword_type($dst$$Register, T_SHORT);
24788 %}
24789 ins_pipe(pipe_slow);
24790 %}
24791
24792 instruct convF2HFAndS2HF(regF dst, regF src)
24793 %{
24794 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
24795 format %{ "convF2HFAndS2HF $dst, $src" %}
24796 ins_encode %{
24797 __ vcvtps2ph($dst$$XMMRegister, $src$$XMMRegister, 0x04, Assembler::AVX_128bit);
24798 %}
24799 ins_pipe(pipe_slow);
24800 %}
24801
24802 instruct convHF2SAndHF2F(regF dst, regF src)
24803 %{
24804 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
24805 format %{ "convHF2SAndHF2F $dst, $src" %}
24806 ins_encode %{
24807 __ vcvtph2ps($dst$$XMMRegister, $src$$XMMRegister, Assembler::AVX_128bit);
24808 %}
24809 ins_pipe(pipe_slow);
24810 %}
24811
24812 instruct scalar_sqrt_HF_reg(regF dst, regF src)
24813 %{
24814 match(Set dst (SqrtHF src));
24815 format %{ "scalar_sqrt_fp16 $dst, $src" %}
24816 ins_encode %{
24817 __ vsqrtsh($dst$$XMMRegister, $src$$XMMRegister);
24818 %}
24819 ins_pipe(pipe_slow);
24820 %}
24821
24822 instruct scalar_binOps_HF_reg(regF dst, regF src1, regF src2)
24823 %{
24824 match(Set dst (AddHF src1 src2));
24825 match(Set dst (DivHF src1 src2));
24826 match(Set dst (MulHF src1 src2));
24827 match(Set dst (SubHF src1 src2));
24828 format %{ "scalar_binop_fp16 $dst, $src1, $src2" %}
24829 ins_encode %{
24830 int opcode = this->ideal_Opcode();
24831 __ efp16sh(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister);
24832 %}
24833 ins_pipe(pipe_slow);
24834 %}
24835
24836 instruct scalar_minmax_HF_reg_avx10_2(regF dst, regF src1, regF src2)
24837 %{
24838 predicate(VM_Version::supports_avx10_2());
24839 match(Set dst (MaxHF src1 src2));
24840 match(Set dst (MinHF src1 src2));
24841
24842 format %{ "scalar_min_max_fp16 $dst, $src1, $src2" %}
24843 ins_encode %{
24844 int opcode = this->ideal_Opcode();
24845 __ sminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, k0);
24846 %}
24847 ins_pipe( pipe_slow );
24848 %}
24849
24850 instruct scalar_minmax_HF_reg(regF dst, regF src1, regF src2, kReg ktmp, regF xtmp1, regF xtmp2)
24851 %{
24852 predicate(!VM_Version::supports_avx10_2());
24853 match(Set dst (MaxHF src1 src2));
24854 match(Set dst (MinHF src1 src2));
24855 effect(TEMP_DEF dst, TEMP ktmp, TEMP xtmp1, TEMP xtmp2);
24856
24857 format %{ "scalar_min_max_fp16 $dst, $src1, $src2\t using $ktmp, $xtmp1 and $xtmp2 as TEMP" %}
24858 ins_encode %{
24859 int opcode = this->ideal_Opcode();
24860 __ sminmax_fp16(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, $ktmp$$KRegister,
24861 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister);
24862 %}
24863 ins_pipe( pipe_slow );
24864 %}
24865
24866 instruct scalar_fma_HF_reg(regF dst, regF src1, regF src2)
24867 %{
24868 match(Set dst (FmaHF src2 (Binary dst src1)));
24869 effect(DEF dst);
24870 format %{ "scalar_fma_fp16 $dst, $src1, $src2\t# $dst = $dst * $src1 + $src2 fma packedH" %}
24871 ins_encode %{
24872 __ vfmadd132sh($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister);
24873 %}
24874 ins_pipe( pipe_slow );
24875 %}
24876
24877
24878 instruct vector_sqrt_HF_reg(vec dst, vec src)
24879 %{
24880 match(Set dst (SqrtVHF src));
24881 format %{ "vector_sqrt_fp16 $dst, $src" %}
24882 ins_encode %{
24883 int vlen_enc = vector_length_encoding(this);
24884 __ evsqrtph($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
24885 %}
24886 ins_pipe(pipe_slow);
24887 %}
24888
24889 instruct vector_sqrt_HF_mem(vec dst, memory src)
24890 %{
24891 match(Set dst (SqrtVHF (VectorReinterpret (LoadVector src))));
24892 format %{ "vector_sqrt_fp16_mem $dst, $src" %}
24893 ins_encode %{
24894 int vlen_enc = vector_length_encoding(this);
24895 __ evsqrtph($dst$$XMMRegister, $src$$Address, vlen_enc);
24896 %}
24897 ins_pipe(pipe_slow);
24898 %}
24899
24900 instruct vector_binOps_HF_reg(vec dst, vec src1, vec src2)
24901 %{
24902 match(Set dst (AddVHF src1 src2));
24903 match(Set dst (DivVHF src1 src2));
24904 match(Set dst (MulVHF src1 src2));
24905 match(Set dst (SubVHF src1 src2));
24906 format %{ "vector_binop_fp16 $dst, $src1, $src2" %}
24907 ins_encode %{
24908 int vlen_enc = vector_length_encoding(this);
24909 int opcode = this->ideal_Opcode();
24910 __ evfp16ph(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister, vlen_enc);
24911 %}
24912 ins_pipe(pipe_slow);
24913 %}
24914
24915
24916 instruct vector_binOps_HF_mem(vec dst, vec src1, memory src2)
24917 %{
24918 match(Set dst (AddVHF src1 (VectorReinterpret (LoadVector src2))));
24919 match(Set dst (DivVHF src1 (VectorReinterpret (LoadVector src2))));
24920 match(Set dst (MulVHF src1 (VectorReinterpret (LoadVector src2))));
24921 match(Set dst (SubVHF src1 (VectorReinterpret (LoadVector src2))));
24922 format %{ "vector_binop_fp16_mem $dst, $src1, $src2" %}
24923 ins_encode %{
24924 int vlen_enc = vector_length_encoding(this);
24925 int opcode = this->ideal_Opcode();
24926 __ evfp16ph(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address, vlen_enc);
24927 %}
24928 ins_pipe(pipe_slow);
24929 %}
24930
24931 instruct vector_fma_HF_reg(vec dst, vec src1, vec src2)
24932 %{
24933 match(Set dst (FmaVHF src2 (Binary dst src1)));
24934 format %{ "vector_fma_fp16 $dst, $src1, $src2\t# $dst = $dst * $src1 + $src2 fma packedH" %}
24935 ins_encode %{
24936 int vlen_enc = vector_length_encoding(this);
24937 __ evfmadd132ph($dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, vlen_enc);
24938 %}
24939 ins_pipe( pipe_slow );
24940 %}
24941
24942 instruct vector_fma_HF_mem(vec dst, memory src1, vec src2)
24943 %{
24944 match(Set dst (FmaVHF src2 (Binary dst (VectorReinterpret (LoadVector src1)))));
24945 format %{ "vector_fma_fp16_mem $dst, $src1, $src2\t# $dst = $dst * $src1 + $src2 fma packedH" %}
24946 ins_encode %{
24947 int vlen_enc = vector_length_encoding(this);
24948 __ evfmadd132ph($dst$$XMMRegister, $src2$$XMMRegister, $src1$$Address, vlen_enc);
24949 %}
24950 ins_pipe( pipe_slow );
24951 %}
24952
24953 instruct vector_minmax_HF_mem_avx10_2(vec dst, vec src1, memory src2)
24954 %{
24955 predicate(VM_Version::supports_avx10_2());
24956 match(Set dst (MinVHF src1 (VectorReinterpret (LoadVector src2))));
24957 match(Set dst (MaxVHF src1 (VectorReinterpret (LoadVector src2))));
24958 format %{ "vector_min_max_fp16_mem $dst, $src1, $src2" %}
24959 ins_encode %{
24960 int vlen_enc = vector_length_encoding(this);
24961 int opcode = this->ideal_Opcode();
24962 __ vminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$Address,
24963 k0, vlen_enc);
24964 %}
24965 ins_pipe( pipe_slow );
24966 %}
24967
24968 instruct vector_minmax_HF_reg_avx10_2(vec dst, vec src1, vec src2)
24969 %{
24970 predicate(VM_Version::supports_avx10_2());
24971 match(Set dst (MinVHF src1 src2));
24972 match(Set dst (MaxVHF src1 src2));
24973 format %{ "vector_min_max_fp16 $dst, $src1, $src2" %}
24974 ins_encode %{
24975 int vlen_enc = vector_length_encoding(this);
24976 int opcode = this->ideal_Opcode();
24977 __ vminmax_fp16_avx10_2(opcode, $dst$$XMMRegister, $src1$$XMMRegister, $src2$$XMMRegister,
24978 k0, vlen_enc);
24979 %}
24980 ins_pipe( pipe_slow );
24981 %}
24982
24983 instruct vector_minmax_HF_reg(vec dst, vec src1, vec src2, kReg ktmp, vec xtmp1, vec xtmp2)
24984 %{
24985 predicate(!VM_Version::supports_avx10_2());
24986 match(Set dst (MinVHF src1 src2));
24987 match(Set dst (MaxVHF src1 src2));
24988 effect(TEMP_DEF dst, TEMP ktmp, TEMP xtmp1, TEMP xtmp2);
24989 format %{ "vector_min_max_fp16 $dst, $src1, $src2\t using $ktmp, $xtmp1 and $xtmp2 as TEMP" %}
24990 ins_encode %{
24991 int vlen_enc = vector_length_encoding(this);
24992 int opcode = this->ideal_Opcode();
24993 __ vminmax_fp16(opcode, $dst$$XMMRegister, $src2$$XMMRegister, $src1$$XMMRegister, $ktmp$$KRegister,
24994 $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, vlen_enc);
24995 %}
24996 ins_pipe( pipe_slow );
24997 %}
24998
24999 //----------PEEPHOLE RULES-----------------------------------------------------
25000 // These must follow all instruction definitions as they use the names
25001 // defined in the instructions definitions.
25002 //
25003 // peeppredicate ( rule_predicate );
25004 // // the predicate unless which the peephole rule will be ignored
25005 //
25006 // peepmatch ( root_instr_name [preceding_instruction]* );
25007 //
25008 // peepprocedure ( procedure_name );
25009 // // provide a procedure name to perform the optimization, the procedure should
25010 // // reside in the architecture dependent peephole file, the method has the
25011 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
25012 // // with the arguments being the basic block, the current node index inside the
25013 // // block, the register allocator, the functions upon invoked return a new node
25014 // // defined in peepreplace, and the rules of the nodes appearing in the
25015 // // corresponding peepmatch, the function return true if successful, else
25016 // // return false
25017 //
25018 // peepconstraint %{
25019 // (instruction_number.operand_name relational_op instruction_number.operand_name
25020 // [, ...] );
25021 // // instruction numbers are zero-based using left to right order in peepmatch
25022 //
25023 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
25024 // // provide an instruction_number.operand_name for each operand that appears
25025 // // in the replacement instruction's match rule
25026 //
25027 // ---------VM FLAGS---------------------------------------------------------
25028 //
25029 // All peephole optimizations can be turned off using -XX:-OptoPeephole
25030 //
25031 // Each peephole rule is given an identifying number starting with zero and
25032 // increasing by one in the order seen by the parser. An individual peephole
25033 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
25034 // on the command-line.
25035 //
25036 // ---------CURRENT LIMITATIONS----------------------------------------------
25037 //
25038 // Only transformations inside a basic block (do we need more for peephole)
25039 //
25040 // ---------EXAMPLE----------------------------------------------------------
25041 //
25042 // // pertinent parts of existing instructions in architecture description
25043 // instruct movI(rRegI dst, rRegI src)
25044 // %{
25045 // match(Set dst (CopyI src));
25046 // %}
25047 //
25048 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
25049 // %{
25050 // match(Set dst (AddI dst src));
25051 // effect(KILL cr);
25052 // %}
25053 //
25054 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
25055 // %{
25056 // match(Set dst (AddI dst src));
25057 // %}
25058 //
25059 // 1. Simple replacement
25060 // - Only match adjacent instructions in same basic block
25061 // - Only equality constraints
25062 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
25063 // - Only one replacement instruction
25064 //
25065 // // Change (inc mov) to lea
25066 // peephole %{
25067 // // lea should only be emitted when beneficial
25068 // peeppredicate( VM_Version::supports_fast_2op_lea() );
25069 // // increment preceded by register-register move
25070 // peepmatch ( incI_rReg movI );
25071 // // require that the destination register of the increment
25072 // // match the destination register of the move
25073 // peepconstraint ( 0.dst == 1.dst );
25074 // // construct a replacement instruction that sets
25075 // // the destination to ( move's source register + one )
25076 // peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
25077 // %}
25078 //
25079 // 2. Procedural replacement
25080 // - More flexible finding relevent nodes
25081 // - More flexible constraints
25082 // - More flexible transformations
25083 // - May utilise architecture-dependent API more effectively
25084 // - Currently only one replacement instruction due to adlc parsing capabilities
25085 //
25086 // // Change (inc mov) to lea
25087 // peephole %{
25088 // // lea should only be emitted when beneficial
25089 // peeppredicate( VM_Version::supports_fast_2op_lea() );
25090 // // the rule numbers of these nodes inside are passed into the function below
25091 // peepmatch ( incI_rReg movI );
25092 // // the method that takes the responsibility of transformation
25093 // peepprocedure ( inc_mov_to_lea );
25094 // // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
25095 // // node is passed into the function above
25096 // peepreplace ( leaI_rReg_immI() );
25097 // %}
25098
25099 // These instructions is not matched by the matcher but used by the peephole
25100 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
25101 %{
25102 predicate(false);
25103 match(Set dst (AddI src1 src2));
25104 format %{ "leal $dst, [$src1 + $src2]" %}
25105 ins_encode %{
25106 Register dst = $dst$$Register;
25107 Register src1 = $src1$$Register;
25108 Register src2 = $src2$$Register;
25109 if (src1 != rbp && src1 != r13) {
25110 __ leal(dst, Address(src1, src2, Address::times_1));
25111 } else {
25112 assert(src2 != rbp && src2 != r13, "");
25113 __ leal(dst, Address(src2, src1, Address::times_1));
25114 }
25115 %}
25116 ins_pipe(ialu_reg_reg);
25117 %}
25118
25119 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
25120 %{
25121 predicate(false);
25122 match(Set dst (AddI src1 src2));
25123 format %{ "leal $dst, [$src1 + $src2]" %}
25124 ins_encode %{
25125 __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
25126 %}
25127 ins_pipe(ialu_reg_reg);
25128 %}
25129
25130 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
25131 %{
25132 predicate(false);
25133 match(Set dst (LShiftI src shift));
25134 format %{ "leal $dst, [$src << $shift]" %}
25135 ins_encode %{
25136 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
25137 Register src = $src$$Register;
25138 if (scale == Address::times_2 && src != rbp && src != r13) {
25139 __ leal($dst$$Register, Address(src, src, Address::times_1));
25140 } else {
25141 __ leal($dst$$Register, Address(noreg, src, scale));
25142 }
25143 %}
25144 ins_pipe(ialu_reg_reg);
25145 %}
25146
25147 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
25148 %{
25149 predicate(false);
25150 match(Set dst (AddL src1 src2));
25151 format %{ "leaq $dst, [$src1 + $src2]" %}
25152 ins_encode %{
25153 Register dst = $dst$$Register;
25154 Register src1 = $src1$$Register;
25155 Register src2 = $src2$$Register;
25156 if (src1 != rbp && src1 != r13) {
25157 __ leaq(dst, Address(src1, src2, Address::times_1));
25158 } else {
25159 assert(src2 != rbp && src2 != r13, "");
25160 __ leaq(dst, Address(src2, src1, Address::times_1));
25161 }
25162 %}
25163 ins_pipe(ialu_reg_reg);
25164 %}
25165
25166 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
25167 %{
25168 predicate(false);
25169 match(Set dst (AddL src1 src2));
25170 format %{ "leaq $dst, [$src1 + $src2]" %}
25171 ins_encode %{
25172 __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
25173 %}
25174 ins_pipe(ialu_reg_reg);
25175 %}
25176
25177 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
25178 %{
25179 predicate(false);
25180 match(Set dst (LShiftL src shift));
25181 format %{ "leaq $dst, [$src << $shift]" %}
25182 ins_encode %{
25183 Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
25184 Register src = $src$$Register;
25185 if (scale == Address::times_2 && src != rbp && src != r13) {
25186 __ leaq($dst$$Register, Address(src, src, Address::times_1));
25187 } else {
25188 __ leaq($dst$$Register, Address(noreg, src, scale));
25189 }
25190 %}
25191 ins_pipe(ialu_reg_reg);
25192 %}
25193
25194 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
25195 // sal}) with lea instructions. The {add, sal} rules are beneficial in
25196 // processors with at least partial ALU support for lea
25197 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
25198 // beneficial for processors with full ALU support
25199 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
25200
25201 peephole
25202 %{
25203 peeppredicate(VM_Version::supports_fast_2op_lea());
25204 peepmatch (addI_rReg);
25205 peepprocedure (lea_coalesce_reg);
25206 peepreplace (leaI_rReg_rReg_peep());
25207 %}
25208
25209 peephole
25210 %{
25211 peeppredicate(VM_Version::supports_fast_2op_lea());
25212 peepmatch (addI_rReg_imm);
25213 peepprocedure (lea_coalesce_imm);
25214 peepreplace (leaI_rReg_immI_peep());
25215 %}
25216
25217 peephole
25218 %{
25219 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25220 VM_Version::is_intel_cascade_lake());
25221 peepmatch (incI_rReg);
25222 peepprocedure (lea_coalesce_imm);
25223 peepreplace (leaI_rReg_immI_peep());
25224 %}
25225
25226 peephole
25227 %{
25228 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25229 VM_Version::is_intel_cascade_lake());
25230 peepmatch (decI_rReg);
25231 peepprocedure (lea_coalesce_imm);
25232 peepreplace (leaI_rReg_immI_peep());
25233 %}
25234
25235 peephole
25236 %{
25237 peeppredicate(VM_Version::supports_fast_2op_lea());
25238 peepmatch (salI_rReg_immI2);
25239 peepprocedure (lea_coalesce_imm);
25240 peepreplace (leaI_rReg_immI2_peep());
25241 %}
25242
25243 peephole
25244 %{
25245 peeppredicate(VM_Version::supports_fast_2op_lea());
25246 peepmatch (addL_rReg);
25247 peepprocedure (lea_coalesce_reg);
25248 peepreplace (leaL_rReg_rReg_peep());
25249 %}
25250
25251 peephole
25252 %{
25253 peeppredicate(VM_Version::supports_fast_2op_lea());
25254 peepmatch (addL_rReg_imm);
25255 peepprocedure (lea_coalesce_imm);
25256 peepreplace (leaL_rReg_immL32_peep());
25257 %}
25258
25259 peephole
25260 %{
25261 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25262 VM_Version::is_intel_cascade_lake());
25263 peepmatch (incL_rReg);
25264 peepprocedure (lea_coalesce_imm);
25265 peepreplace (leaL_rReg_immL32_peep());
25266 %}
25267
25268 peephole
25269 %{
25270 peeppredicate(VM_Version::supports_fast_3op_lea() ||
25271 VM_Version::is_intel_cascade_lake());
25272 peepmatch (decL_rReg);
25273 peepprocedure (lea_coalesce_imm);
25274 peepreplace (leaL_rReg_immL32_peep());
25275 %}
25276
25277 peephole
25278 %{
25279 peeppredicate(VM_Version::supports_fast_2op_lea());
25280 peepmatch (salL_rReg_immI2);
25281 peepprocedure (lea_coalesce_imm);
25282 peepreplace (leaL_rReg_immI2_peep());
25283 %}
25284
25285 peephole
25286 %{
25287 peepmatch (leaPCompressedOopOffset);
25288 peepprocedure (lea_remove_redundant);
25289 %}
25290
25291 peephole
25292 %{
25293 peepmatch (leaP8Narrow);
25294 peepprocedure (lea_remove_redundant);
25295 %}
25296
25297 peephole
25298 %{
25299 peepmatch (leaP32Narrow);
25300 peepprocedure (lea_remove_redundant);
25301 %}
25302
25303 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
25304 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
25305
25306 //int variant
25307 peephole
25308 %{
25309 peepmatch (testI_reg);
25310 peepprocedure (test_may_remove);
25311 %}
25312
25313 //long variant
25314 peephole
25315 %{
25316 peepmatch (testL_reg);
25317 peepprocedure (test_may_remove);
25318 %}
25319
25320
25321 //----------SMARTSPILL RULES---------------------------------------------------
25322 // These must follow all instruction definitions as they use the names
25323 // defined in the instructions definitions.