1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return MacroAssembler::max_trampoline_stub_size();
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 5; // metadata; call dest; trampoline address; trampoline destination; trampoline_owner_metadata
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_CompareAndSwapB:
1512 case Op_CompareAndSwapS:
1513 case Op_GetAndSetI:
1514 case Op_GetAndSetL:
1515 case Op_GetAndSetP:
1516 case Op_GetAndSetN:
1517 case Op_GetAndAddI:
1518 case Op_GetAndAddL:
1519 return true;
1520 case Op_CompareAndExchangeI:
1521 case Op_CompareAndExchangeN:
1522 case Op_CompareAndExchangeB:
1523 case Op_CompareAndExchangeS:
1524 case Op_CompareAndExchangeL:
1525 case Op_CompareAndExchangeP:
1526 case Op_WeakCompareAndSwapB:
1527 case Op_WeakCompareAndSwapS:
1528 case Op_WeakCompareAndSwapI:
1529 case Op_WeakCompareAndSwapL:
1530 case Op_WeakCompareAndSwapP:
1531 case Op_WeakCompareAndSwapN:
1532 return maybe_volatile;
1533 default:
1534 return false;
1535 }
1536 }
1537
1538 // helper to determine the maximum number of Phi nodes we may need to
1539 // traverse when searching from a card mark membar for the merge mem
1540 // feeding a trailing membar or vice versa
1541
1542 // predicates controlling emit of ldr<x>/ldar<x>
1543
1544 bool unnecessary_acquire(const Node *barrier)
1545 {
1546 assert(barrier->is_MemBar(), "expecting a membar");
1547
1548 MemBarNode* mb = barrier->as_MemBar();
1549
1550 if (mb->trailing_load()) {
1551 return true;
1552 }
1553
1554 if (mb->trailing_load_store()) {
1555 Node* load_store = mb->in(MemBarNode::Precedent);
1556 assert(load_store->is_LoadStore(), "unexpected graph shape");
1557 return is_CAS(load_store->Opcode(), true);
1558 }
1559
1560 return false;
1561 }
1562
1563 bool needs_acquiring_load(const Node *n)
1564 {
1565 assert(n->is_Load(), "expecting a load");
1566 LoadNode *ld = n->as_Load();
1567 return ld->is_acquire();
1568 }
1569
1570 bool unnecessary_release(const Node *n)
1571 {
1572 assert((n->is_MemBar() &&
1573 n->Opcode() == Op_MemBarRelease),
1574 "expecting a release membar");
1575
1576 MemBarNode *barrier = n->as_MemBar();
1577 if (!barrier->leading()) {
1578 return false;
1579 } else {
1580 Node* trailing = barrier->trailing_membar();
1581 MemBarNode* trailing_mb = trailing->as_MemBar();
1582 assert(trailing_mb->trailing(), "Not a trailing membar?");
1583 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1584
1585 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1586 if (mem->is_Store()) {
1587 assert(mem->as_Store()->is_release(), "");
1588 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1589 return true;
1590 } else {
1591 assert(mem->is_LoadStore(), "");
1592 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1593 return is_CAS(mem->Opcode(), true);
1594 }
1595 }
1596 return false;
1597 }
1598
1599 bool unnecessary_volatile(const Node *n)
1600 {
1601 // assert n->is_MemBar();
1602 MemBarNode *mbvol = n->as_MemBar();
1603
1604 bool release = mbvol->trailing_store();
1605 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1606 #ifdef ASSERT
1607 if (release) {
1608 Node* leading = mbvol->leading_membar();
1609 assert(leading->Opcode() == Op_MemBarRelease, "");
1610 assert(leading->as_MemBar()->leading_store(), "");
1611 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1612 }
1613 #endif
1614
1615 return release;
1616 }
1617
1618 // predicates controlling emit of str<x>/stlr<x>
1619
1620 bool needs_releasing_store(const Node *n)
1621 {
1622 // assert n->is_Store();
1623 StoreNode *st = n->as_Store();
1624 return st->trailing_membar() != nullptr;
1625 }
1626
1627 // predicate controlling translation of CAS
1628 //
1629 // returns true if CAS needs to use an acquiring load otherwise false
1630
1631 bool needs_acquiring_load_exclusive(const Node *n)
1632 {
1633 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1634 LoadStoreNode* ldst = n->as_LoadStore();
1635 if (is_CAS(n->Opcode(), false)) {
1636 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1637 } else {
1638 return ldst->trailing_membar() != nullptr;
1639 }
1640
1641 // so we can just return true here
1642 return true;
1643 }
1644
1645 #define __ masm->
1646
1647 // advance declarations for helper functions to convert register
1648 // indices to register objects
1649
1650 // the ad file has to provide implementations of certain methods
1651 // expected by the generic code
1652 //
1653 // REQUIRED FUNCTIONALITY
1654
1655 //=============================================================================
1656
1657 // !!!!! Special hack to get all types of calls to specify the byte offset
1658 // from the start of the call to the point where the return address
1659 // will point.
1660
1661 int MachCallStaticJavaNode::ret_addr_offset()
1662 {
1663 // call should be a simple bl
1664 int off = 4;
1665 return off;
1666 }
1667
1668 int MachCallDynamicJavaNode::ret_addr_offset()
1669 {
1670 return 16; // movz, movk, movk, bl
1671 }
1672
1673 int MachCallRuntimeNode::ret_addr_offset() {
1674 // for generated stubs the call will be
1675 // bl(addr)
1676 // or with far branches
1677 // bl(trampoline_stub)
1678 // for real runtime callouts it will be six instructions
1679 // see aarch64_enc_java_to_runtime
1680 // adr(rscratch2, retaddr)
1681 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1682 // lea(rscratch1, RuntimeAddress(addr)
1683 // blr(rscratch1)
1684 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1685 if (cb) {
1686 return 1 * NativeInstruction::instruction_size;
1687 } else {
1688 return 6 * NativeInstruction::instruction_size;
1689 }
1690 }
1691
1692 //=============================================================================
1693
1694 #ifndef PRODUCT
1695 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1696 st->print("BREAKPOINT");
1697 }
1698 #endif
1699
1700 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1701 __ brk(0);
1702 }
1703
1704 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1705 return MachNode::size(ra_);
1706 }
1707
1708 //=============================================================================
1709
1710 #ifndef PRODUCT
1711 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1712 st->print("nop \t# %d bytes pad for loops and calls", _count);
1713 }
1714 #endif
1715
1716 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1717 for (int i = 0; i < _count; i++) {
1718 __ nop();
1719 }
1720 }
1721
1722 uint MachNopNode::size(PhaseRegAlloc*) const {
1723 return _count * NativeInstruction::instruction_size;
1724 }
1725
1726 //=============================================================================
1727 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1728
1729 int ConstantTable::calculate_table_base_offset() const {
1730 return 0; // absolute addressing, no offset
1731 }
1732
1733 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1734 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1735 ShouldNotReachHere();
1736 }
1737
1738 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1739 // Empty encoding
1740 }
1741
1742 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1743 return 0;
1744 }
1745
1746 #ifndef PRODUCT
1747 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1748 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1749 }
1750 #endif
1751
1752 #ifndef PRODUCT
1753 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1754 Compile* C = ra_->C;
1755
1756 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1757
1758 if (C->output()->need_stack_bang(framesize))
1759 st->print("# stack bang size=%d\n\t", framesize);
1760
1761 if (VM_Version::use_rop_protection()) {
1762 st->print("ldr zr, [lr]\n\t");
1763 st->print("paciaz\n\t");
1764 }
1765 if (framesize < ((1 << 9) + 2 * wordSize)) {
1766 st->print("sub sp, sp, #%d\n\t", framesize);
1767 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1768 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1769 } else {
1770 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1771 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1772 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1773 st->print("sub sp, sp, rscratch1");
1774 }
1775 if (C->stub_function() == nullptr) {
1776 st->print("\n\t");
1777 st->print("ldr rscratch1, [guard]\n\t");
1778 st->print("dmb ishld\n\t");
1779 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1780 st->print("cmp rscratch1, rscratch2\n\t");
1781 st->print("b.eq skip");
1782 st->print("\n\t");
1783 st->print("blr #nmethod_entry_barrier_stub\n\t");
1784 st->print("b skip\n\t");
1785 st->print("guard: int\n\t");
1786 st->print("\n\t");
1787 st->print("skip:\n\t");
1788 }
1789 }
1790 #endif
1791
1792 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1793 Compile* C = ra_->C;
1794
1795 // n.b. frame size includes space for return pc and rfp
1796 const int framesize = C->output()->frame_size_in_bytes();
1797
1798 if (C->clinit_barrier_on_entry()) {
1799 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1800
1801 Label L_skip_barrier;
1802
1803 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1804 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1805 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1806 __ bind(L_skip_barrier);
1807 }
1808
1809 if (C->max_vector_size() > 0) {
1810 __ reinitialize_ptrue();
1811 }
1812
1813 int bangsize = C->output()->bang_size_in_bytes();
1814 if (C->output()->need_stack_bang(bangsize))
1815 __ generate_stack_overflow_check(bangsize);
1816
1817 __ build_frame(framesize);
1818
1819 if (C->stub_function() == nullptr) {
1820 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1821 // Dummy labels for just measuring the code size
1822 Label dummy_slow_path;
1823 Label dummy_continuation;
1824 Label dummy_guard;
1825 Label* slow_path = &dummy_slow_path;
1826 Label* continuation = &dummy_continuation;
1827 Label* guard = &dummy_guard;
1828 if (!Compile::current()->output()->in_scratch_emit_size()) {
1829 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1830 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1831 Compile::current()->output()->add_stub(stub);
1832 slow_path = &stub->entry();
1833 continuation = &stub->continuation();
1834 guard = &stub->guard();
1835 }
1836 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1837 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1838 }
1839
1840 if (VerifyStackAtCalls) {
1841 Unimplemented();
1842 }
1843
1844 C->output()->set_frame_complete(__ offset());
1845
1846 if (C->has_mach_constant_base_node()) {
1847 // NOTE: We set the table base offset here because users might be
1848 // emitted before MachConstantBaseNode.
1849 ConstantTable& constant_table = C->output()->constant_table();
1850 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1851 }
1852 }
1853
1854 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1855 {
1856 return MachNode::size(ra_); // too many variables; just compute it
1857 // the hard way
1858 }
1859
1860 int MachPrologNode::reloc() const
1861 {
1862 return 0;
1863 }
1864
1865 //=============================================================================
1866
1867 #ifndef PRODUCT
1868 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1869 Compile* C = ra_->C;
1870 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1871
1872 st->print("# pop frame %d\n\t",framesize);
1873
1874 if (framesize == 0) {
1875 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1876 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1877 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1878 st->print("add sp, sp, #%d\n\t", framesize);
1879 } else {
1880 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1881 st->print("add sp, sp, rscratch1\n\t");
1882 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1883 }
1884 if (VM_Version::use_rop_protection()) {
1885 st->print("autiaz\n\t");
1886 st->print("ldr zr, [lr]\n\t");
1887 }
1888
1889 if (do_polling() && C->is_method_compilation()) {
1890 st->print("# test polling word\n\t");
1891 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1892 st->print("cmp sp, rscratch1\n\t");
1893 st->print("bhi #slow_path");
1894 }
1895 }
1896 #endif
1897
1898 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1899 Compile* C = ra_->C;
1900 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1901
1902 __ remove_frame(framesize);
1903
1904 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1905 __ reserved_stack_check();
1906 }
1907
1908 if (do_polling() && C->is_method_compilation()) {
1909 Label dummy_label;
1910 Label* code_stub = &dummy_label;
1911 if (!C->output()->in_scratch_emit_size()) {
1912 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1913 C->output()->add_stub(stub);
1914 code_stub = &stub->entry();
1915 }
1916 __ relocate(relocInfo::poll_return_type);
1917 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1918 }
1919 }
1920
1921 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1922 // Variable size. Determine dynamically.
1923 return MachNode::size(ra_);
1924 }
1925
1926 int MachEpilogNode::reloc() const {
1927 // Return number of relocatable values contained in this instruction.
1928 return 1; // 1 for polling page.
1929 }
1930
1931 const Pipeline * MachEpilogNode::pipeline() const {
1932 return MachNode::pipeline_class();
1933 }
1934
1935 //=============================================================================
1936
1937 static enum RC rc_class(OptoReg::Name reg) {
1938
1939 if (reg == OptoReg::Bad) {
1940 return rc_bad;
1941 }
1942
1943 // we have 32 int registers * 2 halves
1944 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1945
1946 if (reg < slots_of_int_registers) {
1947 return rc_int;
1948 }
1949
1950 // we have 32 float register * 8 halves
1951 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1952 if (reg < slots_of_int_registers + slots_of_float_registers) {
1953 return rc_float;
1954 }
1955
1956 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1957 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1958 return rc_predicate;
1959 }
1960
1961 // Between predicate regs & stack is the flags.
1962 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1963
1964 return rc_stack;
1965 }
1966
1967 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1968 Compile* C = ra_->C;
1969
1970 // Get registers to move.
1971 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1972 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1973 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1974 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1975
1976 enum RC src_hi_rc = rc_class(src_hi);
1977 enum RC src_lo_rc = rc_class(src_lo);
1978 enum RC dst_hi_rc = rc_class(dst_hi);
1979 enum RC dst_lo_rc = rc_class(dst_lo);
1980
1981 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1982
1983 if (src_hi != OptoReg::Bad && !bottom_type()->isa_pvectmask()) {
1984 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1985 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1986 "expected aligned-adjacent pairs");
1987 }
1988
1989 if (src_lo == dst_lo && src_hi == dst_hi) {
1990 return 0; // Self copy, no move.
1991 }
1992
1993 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1994 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1995 int src_offset = ra_->reg2offset(src_lo);
1996 int dst_offset = ra_->reg2offset(dst_lo);
1997
1998 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
1999 uint ireg = ideal_reg();
2000 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
2001 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
2002 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
2003 if (ireg == Op_VecA && masm) {
2004 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2005 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2006 // stack->stack
2007 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2008 sve_vector_reg_size_in_bytes);
2009 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2010 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2011 sve_vector_reg_size_in_bytes);
2012 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2013 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2016 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 as_FloatRegister(Matcher::_regEncode[src_lo]),
2018 as_FloatRegister(Matcher::_regEncode[src_lo]));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 } else if (masm) {
2023 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2024 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2025 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2026 // stack->stack
2027 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2028 if (ireg == Op_VecD) {
2029 __ unspill(rscratch1, true, src_offset);
2030 __ spill(rscratch1, true, dst_offset);
2031 } else {
2032 __ spill_copy128(src_offset, dst_offset);
2033 }
2034 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2035 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2036 ireg == Op_VecD ? __ T8B : __ T16B,
2037 as_FloatRegister(Matcher::_regEncode[src_lo]));
2038 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2039 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2040 ireg == Op_VecD ? __ D : __ Q,
2041 ra_->reg2offset(dst_lo));
2042 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2043 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2044 ireg == Op_VecD ? __ D : __ Q,
2045 ra_->reg2offset(src_lo));
2046 } else {
2047 ShouldNotReachHere();
2048 }
2049 }
2050 } else if (masm) {
2051 switch (src_lo_rc) {
2052 case rc_int:
2053 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2054 if (is64) {
2055 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2056 as_Register(Matcher::_regEncode[src_lo]));
2057 } else {
2058 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2059 as_Register(Matcher::_regEncode[src_lo]));
2060 }
2061 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2062 if (is64) {
2063 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2064 as_Register(Matcher::_regEncode[src_lo]));
2065 } else {
2066 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2067 as_Register(Matcher::_regEncode[src_lo]));
2068 }
2069 } else { // gpr --> stack spill
2070 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2071 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2072 }
2073 break;
2074 case rc_float:
2075 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2076 if (is64) {
2077 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2078 as_FloatRegister(Matcher::_regEncode[src_lo]));
2079 } else {
2080 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2081 as_FloatRegister(Matcher::_regEncode[src_lo]));
2082 }
2083 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2084 if (is64) {
2085 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2086 as_FloatRegister(Matcher::_regEncode[src_lo]));
2087 } else {
2088 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2089 as_FloatRegister(Matcher::_regEncode[src_lo]));
2090 }
2091 } else { // fpr --> stack spill
2092 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2093 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2094 is64 ? __ D : __ S, dst_offset);
2095 }
2096 break;
2097 case rc_stack:
2098 if (dst_lo_rc == rc_int) { // stack --> gpr load
2099 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2100 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2101 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2102 is64 ? __ D : __ S, src_offset);
2103 } else if (dst_lo_rc == rc_predicate) {
2104 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2105 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2106 } else { // stack --> stack copy
2107 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2108 if (ideal_reg() == Op_RegVectMask) {
2109 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2110 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2111 } else {
2112 __ unspill(rscratch1, is64, src_offset);
2113 __ spill(rscratch1, is64, dst_offset);
2114 }
2115 }
2116 break;
2117 case rc_predicate:
2118 if (dst_lo_rc == rc_predicate) {
2119 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2120 } else if (dst_lo_rc == rc_stack) {
2121 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2122 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2123 } else {
2124 assert(false, "bad src and dst rc_class combination.");
2125 ShouldNotReachHere();
2126 }
2127 break;
2128 default:
2129 assert(false, "bad rc_class for spill");
2130 ShouldNotReachHere();
2131 }
2132 }
2133
2134 if (st) {
2135 st->print("spill ");
2136 if (src_lo_rc == rc_stack) {
2137 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2138 } else {
2139 st->print("%s -> ", Matcher::regName[src_lo]);
2140 }
2141 if (dst_lo_rc == rc_stack) {
2142 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2143 } else {
2144 st->print("%s", Matcher::regName[dst_lo]);
2145 }
2146 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
2147 int vsize = 0;
2148 switch (ideal_reg()) {
2149 case Op_VecD:
2150 vsize = 64;
2151 break;
2152 case Op_VecX:
2153 vsize = 128;
2154 break;
2155 case Op_VecA:
2156 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2157 break;
2158 default:
2159 assert(false, "bad register type for spill");
2160 ShouldNotReachHere();
2161 }
2162 st->print("\t# vector spill size = %d", vsize);
2163 } else if (ideal_reg() == Op_RegVectMask) {
2164 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2165 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2166 st->print("\t# predicate spill size = %d", vsize);
2167 } else {
2168 st->print("\t# spill size = %d", is64 ? 64 : 32);
2169 }
2170 }
2171
2172 return 0;
2173
2174 }
2175
2176 #ifndef PRODUCT
2177 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2178 if (!ra_)
2179 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2180 else
2181 implementation(nullptr, ra_, false, st);
2182 }
2183 #endif
2184
2185 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2186 implementation(masm, ra_, false, nullptr);
2187 }
2188
2189 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2190 return MachNode::size(ra_);
2191 }
2192
2193 //=============================================================================
2194
2195 #ifndef PRODUCT
2196 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2197 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2198 int reg = ra_->get_reg_first(this);
2199 st->print("add %s, rsp, #%d]\t# box lock",
2200 Matcher::regName[reg], offset);
2201 }
2202 #endif
2203
2204 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2205 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2206 int reg = ra_->get_encode(this);
2207
2208 // This add will handle any 24-bit signed offset. 24 bits allows an
2209 // 8 megabyte stack frame.
2210 __ add(as_Register(reg), sp, offset);
2211 }
2212
2213 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2214 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2215 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2216
2217 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2218 return NativeInstruction::instruction_size;
2219 } else {
2220 return 2 * NativeInstruction::instruction_size;
2221 }
2222 }
2223
2224 //=============================================================================
2225
2226 #ifndef PRODUCT
2227 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2228 {
2229 st->print_cr("# MachUEPNode");
2230 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2231 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2232 st->print_cr("\tcmpw rscratch1, r10");
2233 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2234 }
2235 #endif
2236
2237 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2238 {
2239 __ ic_check(InteriorEntryAlignment);
2240 }
2241
2242 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2243 {
2244 return MachNode::size(ra_);
2245 }
2246
2247 // REQUIRED EMIT CODE
2248
2249 //=============================================================================
2250
2251 // Emit deopt handler code.
2252 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2253 {
2254 // Note that the code buffer's insts_mark is always relative to insts.
2255 // That's why we must use the macroassembler to generate a handler.
2256 address base = __ start_a_stub(size_deopt_handler());
2257 if (base == nullptr) {
2258 ciEnv::current()->record_failure("CodeCache is full");
2259 return 0; // CodeBuffer::expand failed
2260 }
2261
2262 int offset = __ offset();
2263 Label start;
2264 __ bind(start);
2265 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2266
2267 int entry_offset = __ offset();
2268 __ b(start);
2269
2270 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2271 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2272 "out of bounds read in post-call NOP check");
2273 __ end_a_stub();
2274 return entry_offset;
2275 }
2276
2277 // REQUIRED MATCHER CODE
2278
2279 //=============================================================================
2280
2281 bool Matcher::match_rule_supported(int opcode) {
2282 if (!has_match_rule(opcode))
2283 return false;
2284
2285 switch (opcode) {
2286 case Op_OnSpinWait:
2287 return VM_Version::supports_on_spin_wait();
2288 case Op_CacheWB:
2289 case Op_CacheWBPreSync:
2290 case Op_CacheWBPostSync:
2291 if (!VM_Version::supports_data_cache_line_flush()) {
2292 return false;
2293 }
2294 break;
2295 case Op_ExpandBits:
2296 case Op_CompressBits:
2297 if (!VM_Version::supports_svebitperm()) {
2298 return false;
2299 }
2300 break;
2301 case Op_FmaF:
2302 case Op_FmaD:
2303 case Op_FmaVF:
2304 case Op_FmaVD:
2305 if (!UseFMA) {
2306 return false;
2307 }
2308 break;
2309 case Op_FmaHF:
2310 // UseFMA flag also needs to be checked along with FEAT_FP16
2311 if (!UseFMA || !is_feat_fp16_supported()) {
2312 return false;
2313 }
2314 break;
2315 case Op_AddHF:
2316 case Op_SubHF:
2317 case Op_MulHF:
2318 case Op_DivHF:
2319 case Op_MinHF:
2320 case Op_MaxHF:
2321 case Op_SqrtHF:
2322 // Half-precision floating point scalar operations require FEAT_FP16
2323 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2324 // features are supported.
2325 if (!is_feat_fp16_supported()) {
2326 return false;
2327 }
2328 break;
2329 }
2330
2331 return true; // Per default match rules are supported.
2332 }
2333
2334 const RegMask* Matcher::predicate_reg_mask(void) {
2335 return &_PR_REG_mask;
2336 }
2337
2338 bool Matcher::supports_vector_calling_convention(void) {
2339 return EnableVectorSupport;
2340 }
2341
2342 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2343 assert(EnableVectorSupport, "sanity");
2344 int lo = V0_num;
2345 int hi = V0_H_num;
2346 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2347 hi = V0_K_num;
2348 }
2349 return OptoRegPair(hi, lo);
2350 }
2351
2352 // Is this branch offset short enough that a short branch can be used?
2353 //
2354 // NOTE: If the platform does not provide any short branch variants, then
2355 // this method should return false for offset 0.
2356 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2357 // The passed offset is relative to address of the branch.
2358
2359 return (-32768 <= offset && offset < 32768);
2360 }
2361
2362 // Vector width in bytes.
2363 int Matcher::vector_width_in_bytes(BasicType bt) {
2364 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2365 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2366 // Minimum 2 values in vector
2367 if (size < 2*type2aelembytes(bt)) size = 0;
2368 // But never < 4
2369 if (size < 4) size = 0;
2370 return size;
2371 }
2372
2373 // Limits on vector size (number of elements) loaded into vector.
2374 int Matcher::max_vector_size(const BasicType bt) {
2375 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2376 }
2377
2378 int Matcher::min_vector_size(const BasicType bt) {
2379 // Usually, the shortest vector length supported by AArch64 ISA and
2380 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2381 // vectors in a few special cases.
2382 int size;
2383 switch(bt) {
2384 case T_BOOLEAN:
2385 // Load/store a vector mask with only 2 elements for vector types
2386 // such as "2I/2F/2L/2D".
2387 size = 2;
2388 break;
2389 case T_BYTE:
2390 // Generate a "4B" vector, to support vector cast between "8B/16B"
2391 // and "4S/4I/4L/4F/4D".
2392 size = 4;
2393 break;
2394 case T_SHORT:
2395 // Generate a "2S" vector, to support vector cast between "4S/8S"
2396 // and "2I/2L/2F/2D".
2397 size = 2;
2398 break;
2399 default:
2400 // Limit the min vector length to 64-bit.
2401 size = 8 / type2aelembytes(bt);
2402 // The number of elements in a vector should be at least 2.
2403 size = MAX2(size, 2);
2404 }
2405
2406 int max_size = max_vector_size(bt);
2407 return MIN2(size, max_size);
2408 }
2409
2410 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2411 return Matcher::max_vector_size(bt);
2412 }
2413
2414 // Actual max scalable vector register length.
2415 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2416 return Matcher::max_vector_size(bt);
2417 }
2418
2419 // Vector ideal reg.
2420 uint Matcher::vector_ideal_reg(int len) {
2421 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2422 return Op_VecA;
2423 }
2424 switch(len) {
2425 // For 16-bit/32-bit mask vector, reuse VecD.
2426 case 2:
2427 case 4:
2428 case 8: return Op_VecD;
2429 case 16: return Op_VecX;
2430 }
2431 ShouldNotReachHere();
2432 return 0;
2433 }
2434
2435 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2436 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2437 switch (ideal_reg) {
2438 case Op_VecA: return new vecAOper();
2439 case Op_VecD: return new vecDOper();
2440 case Op_VecX: return new vecXOper();
2441 }
2442 ShouldNotReachHere();
2443 return nullptr;
2444 }
2445
2446 bool Matcher::is_reg2reg_move(MachNode* m) {
2447 return false;
2448 }
2449
2450 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2451 return false;
2452 }
2453
2454 bool Matcher::is_generic_vector(MachOper* opnd) {
2455 return opnd->opcode() == VREG;
2456 }
2457
2458 #ifdef ASSERT
2459 // Return whether or not this register is ever used as an argument.
2460 bool Matcher::can_be_java_arg(int reg)
2461 {
2462 return
2463 reg == R0_num || reg == R0_H_num ||
2464 reg == R1_num || reg == R1_H_num ||
2465 reg == R2_num || reg == R2_H_num ||
2466 reg == R3_num || reg == R3_H_num ||
2467 reg == R4_num || reg == R4_H_num ||
2468 reg == R5_num || reg == R5_H_num ||
2469 reg == R6_num || reg == R6_H_num ||
2470 reg == R7_num || reg == R7_H_num ||
2471 reg == V0_num || reg == V0_H_num ||
2472 reg == V1_num || reg == V1_H_num ||
2473 reg == V2_num || reg == V2_H_num ||
2474 reg == V3_num || reg == V3_H_num ||
2475 reg == V4_num || reg == V4_H_num ||
2476 reg == V5_num || reg == V5_H_num ||
2477 reg == V6_num || reg == V6_H_num ||
2478 reg == V7_num || reg == V7_H_num;
2479 }
2480 #endif
2481
2482 uint Matcher::int_pressure_limit()
2483 {
2484 // JDK-8183543: When taking the number of available registers as int
2485 // register pressure threshold, the jtreg test:
2486 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2487 // failed due to C2 compilation failure with
2488 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2489 //
2490 // A derived pointer is live at CallNode and then is flagged by RA
2491 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2492 // derived pointers and lastly fail to spill after reaching maximum
2493 // number of iterations. Lowering the default pressure threshold to
2494 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2495 // a high register pressure area of the code so that split_DEF can
2496 // generate DefinitionSpillCopy for the derived pointer.
2497 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2498 if (!PreserveFramePointer) {
2499 // When PreserveFramePointer is off, frame pointer is allocatable,
2500 // but different from other SOC registers, it is excluded from
2501 // fatproj's mask because its save type is No-Save. Decrease 1 to
2502 // ensure high pressure at fatproj when PreserveFramePointer is off.
2503 // See check_pressure_at_fatproj().
2504 default_int_pressure_threshold--;
2505 }
2506 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2507 }
2508
2509 uint Matcher::float_pressure_limit()
2510 {
2511 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2512 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2513 }
2514
2515 const RegMask& Matcher::firstI_proj_mask() {
2516 ShouldNotReachHere();
2517 return RegMask::EMPTY;
2518 }
2519
2520 // Register for the second projection of an int pair
2521 const RegMask& Matcher::secondI_proj_mask() {
2522 ShouldNotReachHere();
2523 return RegMask::EMPTY;
2524 }
2525
2526 // Register for the first projection of a long pair
2527 const RegMask& Matcher::firstL_proj_mask() {
2528 ShouldNotReachHere();
2529 return RegMask::EMPTY;
2530 }
2531
2532 // Register for the second projection of a long pair
2533 const RegMask& Matcher::secondL_proj_mask() {
2534 ShouldNotReachHere();
2535 return RegMask::EMPTY;
2536 }
2537
2538 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2539 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2540 Node* u = addp->fast_out(i);
2541 if (u->is_LoadStore()) {
2542 // On AArch64, LoadStoreNodes (i.e. compare and swap
2543 // instructions) only take register indirect as an operand, so
2544 // any attempt to use an AddPNode as an input to a LoadStoreNode
2545 // must fail.
2546 return false;
2547 }
2548 if (u->is_Mem()) {
2549 int opsize = u->as_Mem()->memory_size();
2550 assert(opsize > 0, "unexpected memory operand size");
2551 if (u->as_Mem()->memory_size() != (1<<shift)) {
2552 return false;
2553 }
2554 }
2555 }
2556 return true;
2557 }
2558
2559 // Convert BoolTest condition to Assembler condition.
2560 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2561 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2562 Assembler::Condition result;
2563 switch(cond) {
2564 case BoolTest::eq:
2565 result = Assembler::EQ; break;
2566 case BoolTest::ne:
2567 result = Assembler::NE; break;
2568 case BoolTest::le:
2569 result = Assembler::LE; break;
2570 case BoolTest::ge:
2571 result = Assembler::GE; break;
2572 case BoolTest::lt:
2573 result = Assembler::LT; break;
2574 case BoolTest::gt:
2575 result = Assembler::GT; break;
2576 case BoolTest::ule:
2577 result = Assembler::LS; break;
2578 case BoolTest::uge:
2579 result = Assembler::HS; break;
2580 case BoolTest::ult:
2581 result = Assembler::LO; break;
2582 case BoolTest::ugt:
2583 result = Assembler::HI; break;
2584 case BoolTest::overflow:
2585 result = Assembler::VS; break;
2586 case BoolTest::no_overflow:
2587 result = Assembler::VC; break;
2588 default:
2589 ShouldNotReachHere();
2590 return Assembler::Condition(-1);
2591 }
2592
2593 // Check conversion
2594 if (cond & BoolTest::unsigned_compare) {
2595 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2596 } else {
2597 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2598 }
2599
2600 return result;
2601 }
2602
2603 // Binary src (Replicate con)
2604 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2605 if (n == nullptr || m == nullptr) {
2606 return false;
2607 }
2608
2609 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2610 return false;
2611 }
2612
2613 Node* imm_node = m->in(1);
2614 if (!imm_node->is_Con()) {
2615 return false;
2616 }
2617
2618 const Type* t = imm_node->bottom_type();
2619 if (!(t->isa_int() || t->isa_long())) {
2620 return false;
2621 }
2622
2623 switch (n->Opcode()) {
2624 case Op_AndV:
2625 case Op_OrV:
2626 case Op_XorV: {
2627 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2628 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2629 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2630 }
2631 case Op_AddVB:
2632 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2633 case Op_AddVS:
2634 case Op_AddVI:
2635 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2636 case Op_AddVL:
2637 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2638 default:
2639 return false;
2640 }
2641 }
2642
2643 // (XorV src (Replicate m1))
2644 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2645 if (n != nullptr && m != nullptr) {
2646 return n->Opcode() == Op_XorV &&
2647 VectorNode::is_all_ones_vector(m);
2648 }
2649 return false;
2650 }
2651
2652 // Returns true if (n, m) matches "(XorVMask vm2 (MaskAll m1))" and that XorVMask
2653 // is used only by an AndVMask. In that case, cloning m (the MaskAll) lets the
2654 // matcher avoid sharing the MaskAll node and subsume the pattern into rule:
2655 // "(AndVMask vm1 (XorVMask vm2 (MaskAll m1)))".
2656 //
2657 // Limitation: the "andNot" rule still cannot be matched if "m" has other
2658 // uses outside this pattern.
2659 static bool is_vector_mask_not_operand_in_andnot_pattern(Node* n, Node* m) {
2660 if (n == nullptr || m == nullptr) {
2661 return false;
2662 }
2663
2664 if (VectorNode::is_all_ones_vector(m) &&
2665 n->Opcode() == Op_XorVMask &&
2666 n->outcnt() == 1 &&
2667 n->unique_out()->Opcode() == Op_AndVMask) {
2668 // If another input of the AndVMask is also a mask-not pattern that would
2669 // qualify for the `maskAll` cloning, do not clone the "maskAll" here,
2670 // because the match rule can only consume one such pattern.
2671 Node* use = n->unique_out();
2672 Node* other_input = use->in(1) == n ? use->in(2) : use->in(1);
2673 return !VectorNode::is_vectormask_bitwise_not_pattern(other_input);
2674 }
2675 return false;
2676 }
2677
2678 // Should the matcher clone input 'm' of node 'n'?
2679 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2680 if (is_vshift_con_pattern(n, m) ||
2681 is_vector_bitwise_not_pattern(n, m) ||
2682 is_vector_mask_not_operand_in_andnot_pattern(n, m) ||
2683 is_valid_sve_arith_imm_pattern(n, m) ||
2684 is_encode_and_store_pattern(n, m)) {
2685 mstack.push(m, Visit);
2686 return true;
2687 }
2688 return false;
2689 }
2690
2691 // Should the Matcher clone shifts on addressing modes, expecting them
2692 // to be subsumed into complex addressing expressions or compute them
2693 // into registers?
2694 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2695
2696 // Loads and stores with indirect memory input (e.g., volatile loads and
2697 // stores) do not subsume the input into complex addressing expressions. If
2698 // the addressing expression is input to at least one such load or store, do
2699 // not clone the addressing expression. Query needs_acquiring_load and
2700 // needs_releasing_store as a proxy for indirect memory input, as it is not
2701 // possible to directly query for indirect memory input at this stage.
2702 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2703 Node* n = m->fast_out(i);
2704 if (n->is_Load() && needs_acquiring_load(n)) {
2705 return false;
2706 }
2707 if (n->is_Store() && needs_releasing_store(n)) {
2708 return false;
2709 }
2710 }
2711
2712 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2713 return true;
2714 }
2715
2716 Node *off = m->in(AddPNode::Offset);
2717 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2718 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2719 // Are there other uses besides address expressions?
2720 !is_visited(off)) {
2721 address_visited.set(off->_idx); // Flag as address_visited
2722 mstack.push(off->in(2), Visit);
2723 Node *conv = off->in(1);
2724 if (conv->Opcode() == Op_ConvI2L &&
2725 // Are there other uses besides address expressions?
2726 !is_visited(conv)) {
2727 address_visited.set(conv->_idx); // Flag as address_visited
2728 mstack.push(conv->in(1), Pre_Visit);
2729 } else {
2730 mstack.push(conv, Pre_Visit);
2731 }
2732 address_visited.test_set(m->_idx); // Flag as address_visited
2733 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2734 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2735 return true;
2736 } else if (off->Opcode() == Op_ConvI2L &&
2737 // Are there other uses besides address expressions?
2738 !is_visited(off)) {
2739 address_visited.test_set(m->_idx); // Flag as address_visited
2740 address_visited.set(off->_idx); // Flag as address_visited
2741 mstack.push(off->in(1), Pre_Visit);
2742 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2743 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2744 return true;
2745 }
2746 return false;
2747 }
2748
2749 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2750 { \
2751 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2752 guarantee(DISP == 0, "mode not permitted for volatile"); \
2753 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2754 __ INSN(REG, as_Register(BASE)); \
2755 }
2756
2757
2758 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2759 {
2760 Address::extend scale;
2761
2762 // Hooboy, this is fugly. We need a way to communicate to the
2763 // encoder that the index needs to be sign extended, so we have to
2764 // enumerate all the cases.
2765 switch (opcode) {
2766 case INDINDEXSCALEDI2L:
2767 case INDINDEXSCALEDI2LN:
2768 case INDINDEXI2L:
2769 case INDINDEXI2LN:
2770 scale = Address::sxtw(size);
2771 break;
2772 default:
2773 scale = Address::lsl(size);
2774 }
2775
2776 if (index == -1) {
2777 return Address(base, disp);
2778 } else {
2779 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2780 return Address(base, as_Register(index), scale);
2781 }
2782 }
2783
2784
2785 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2786 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2787 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2788 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2789 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2790
2791 // Used for all non-volatile memory accesses. The use of
2792 // $mem->opcode() to discover whether this pattern uses sign-extended
2793 // offsets is something of a kludge.
2794 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2795 Register reg, int opcode,
2796 Register base, int index, int scale, int disp,
2797 int size_in_memory)
2798 {
2799 Address addr = mem2address(opcode, base, index, scale, disp);
2800 if (addr.getMode() == Address::base_plus_offset) {
2801 /* Fix up any out-of-range offsets. */
2802 assert_different_registers(rscratch1, base);
2803 assert_different_registers(rscratch1, reg);
2804 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2805 }
2806 (masm->*insn)(reg, addr);
2807 }
2808
2809 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2810 FloatRegister reg, int opcode,
2811 Register base, int index, int size, int disp,
2812 int size_in_memory)
2813 {
2814 Address::extend scale;
2815
2816 switch (opcode) {
2817 case INDINDEXSCALEDI2L:
2818 case INDINDEXSCALEDI2LN:
2819 scale = Address::sxtw(size);
2820 break;
2821 default:
2822 scale = Address::lsl(size);
2823 }
2824
2825 if (index == -1) {
2826 // Fix up any out-of-range offsets.
2827 assert_different_registers(rscratch1, base);
2828 Address addr = Address(base, disp);
2829 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2830 (masm->*insn)(reg, addr);
2831 } else {
2832 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2833 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2834 }
2835 }
2836
2837 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2838 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2839 int opcode, Register base, int index, int size, int disp)
2840 {
2841 if (index == -1) {
2842 (masm->*insn)(reg, T, Address(base, disp));
2843 } else {
2844 assert(disp == 0, "unsupported address mode");
2845 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2846 }
2847 }
2848
2849 %}
2850
2851
2852
2853 //----------ENCODING BLOCK-----------------------------------------------------
2854 // This block specifies the encoding classes used by the compiler to
2855 // output byte streams. Encoding classes are parameterized macros
2856 // used by Machine Instruction Nodes in order to generate the bit
2857 // encoding of the instruction. Operands specify their base encoding
2858 // interface with the interface keyword. There are currently
2859 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2860 // COND_INTER. REG_INTER causes an operand to generate a function
2861 // which returns its register number when queried. CONST_INTER causes
2862 // an operand to generate a function which returns the value of the
2863 // constant when queried. MEMORY_INTER causes an operand to generate
2864 // four functions which return the Base Register, the Index Register,
2865 // the Scale Value, and the Offset Value of the operand when queried.
2866 // COND_INTER causes an operand to generate six functions which return
2867 // the encoding code (ie - encoding bits for the instruction)
2868 // associated with each basic boolean condition for a conditional
2869 // instruction.
2870 //
2871 // Instructions specify two basic values for encoding. Again, a
2872 // function is available to check if the constant displacement is an
2873 // oop. They use the ins_encode keyword to specify their encoding
2874 // classes (which must be a sequence of enc_class names, and their
2875 // parameters, specified in the encoding block), and they use the
2876 // opcode keyword to specify, in order, their primary, secondary, and
2877 // tertiary opcode. Only the opcode sections which a particular
2878 // instruction needs for encoding need to be specified.
2879 encode %{
2880 // Build emit functions for each basic byte or larger field in the
2881 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2882 // from C++ code in the enc_class source block. Emit functions will
2883 // live in the main source block for now. In future, we can
2884 // generalize this by adding a syntax that specifies the sizes of
2885 // fields in an order, so that the adlc can build the emit functions
2886 // automagically
2887
2888 // catch all for unimplemented encodings
2889 enc_class enc_unimplemented %{
2890 __ unimplemented("C2 catch all");
2891 %}
2892
2893 // BEGIN Non-volatile memory access
2894
2895 // This encoding class is generated automatically from ad_encode.m4.
2896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2897 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2898 Register dst_reg = as_Register($dst$$reg);
2899 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2900 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2901 %}
2902
2903 // This encoding class is generated automatically from ad_encode.m4.
2904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2905 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2906 Register dst_reg = as_Register($dst$$reg);
2907 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2908 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2909 %}
2910
2911 // This encoding class is generated automatically from ad_encode.m4.
2912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2913 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2914 Register dst_reg = as_Register($dst$$reg);
2915 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2916 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2917 %}
2918
2919 // This encoding class is generated automatically from ad_encode.m4.
2920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2921 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2922 Register dst_reg = as_Register($dst$$reg);
2923 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2924 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2925 %}
2926
2927 // This encoding class is generated automatically from ad_encode.m4.
2928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2929 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2930 Register dst_reg = as_Register($dst$$reg);
2931 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2932 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2933 %}
2934
2935 // This encoding class is generated automatically from ad_encode.m4.
2936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2937 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2938 Register dst_reg = as_Register($dst$$reg);
2939 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2940 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2941 %}
2942
2943 // This encoding class is generated automatically from ad_encode.m4.
2944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2945 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2946 Register dst_reg = as_Register($dst$$reg);
2947 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2948 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2949 %}
2950
2951 // This encoding class is generated automatically from ad_encode.m4.
2952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2953 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2954 Register dst_reg = as_Register($dst$$reg);
2955 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2956 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2957 %}
2958
2959 // This encoding class is generated automatically from ad_encode.m4.
2960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2961 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2962 Register dst_reg = as_Register($dst$$reg);
2963 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2964 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2965 %}
2966
2967 // This encoding class is generated automatically from ad_encode.m4.
2968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2969 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2970 Register dst_reg = as_Register($dst$$reg);
2971 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2972 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2973 %}
2974
2975 // This encoding class is generated automatically from ad_encode.m4.
2976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2977 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2978 Register dst_reg = as_Register($dst$$reg);
2979 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2980 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2981 %}
2982
2983 // This encoding class is generated automatically from ad_encode.m4.
2984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2985 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2986 Register dst_reg = as_Register($dst$$reg);
2987 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2988 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2989 %}
2990
2991 // This encoding class is generated automatically from ad_encode.m4.
2992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2993 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2994 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2995 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2996 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2997 %}
2998
2999 // This encoding class is generated automatically from ad_encode.m4.
3000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3001 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
3002 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3003 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
3004 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3005 %}
3006
3007 // This encoding class is generated automatically from ad_encode.m4.
3008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3009 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3010 Register src_reg = as_Register($src$$reg);
3011 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3012 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3013 %}
3014
3015 // This encoding class is generated automatically from ad_encode.m4.
3016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3017 enc_class aarch64_enc_strb0(memory1 mem) %{
3018 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3019 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3020 %}
3021
3022 // This encoding class is generated automatically from ad_encode.m4.
3023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3024 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3025 Register src_reg = as_Register($src$$reg);
3026 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3027 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3028 %}
3029
3030 // This encoding class is generated automatically from ad_encode.m4.
3031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3032 enc_class aarch64_enc_strh0(memory2 mem) %{
3033 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3034 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3035 %}
3036
3037 // This encoding class is generated automatically from ad_encode.m4.
3038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3039 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3040 Register src_reg = as_Register($src$$reg);
3041 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3042 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3043 %}
3044
3045 // This encoding class is generated automatically from ad_encode.m4.
3046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3047 enc_class aarch64_enc_strw0(memory4 mem) %{
3048 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3049 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3050 %}
3051
3052 // This encoding class is generated automatically from ad_encode.m4.
3053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3054 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3055 Register src_reg = as_Register($src$$reg);
3056 // we sometimes get asked to store the stack pointer into the
3057 // current thread -- we cannot do that directly on AArch64
3058 if (src_reg == r31_sp) {
3059 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3060 __ mov(rscratch2, sp);
3061 src_reg = rscratch2;
3062 }
3063 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3064 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3065 %}
3066
3067 // This encoding class is generated automatically from ad_encode.m4.
3068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3069 enc_class aarch64_enc_str0(memory8 mem) %{
3070 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3071 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3072 %}
3073
3074 // This encoding class is generated automatically from ad_encode.m4.
3075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3076 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3077 FloatRegister src_reg = as_FloatRegister($src$$reg);
3078 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3079 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3080 %}
3081
3082 // This encoding class is generated automatically from ad_encode.m4.
3083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3084 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3085 FloatRegister src_reg = as_FloatRegister($src$$reg);
3086 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3087 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3088 %}
3089
3090 // This encoding class is generated automatically from ad_encode.m4.
3091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3092 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3093 __ membar(Assembler::StoreStore);
3094 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3095 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3096 %}
3097
3098 // END Non-volatile memory access
3099
3100 // Vector loads and stores
3101 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3102 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3103 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3104 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3105 %}
3106
3107 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3108 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3109 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3110 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3111 %}
3112
3113 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3114 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3115 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3116 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3117 %}
3118
3119 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3120 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3121 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3122 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3123 %}
3124
3125 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3126 FloatRegister src_reg = as_FloatRegister($src$$reg);
3127 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3128 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3129 %}
3130
3131 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3132 FloatRegister src_reg = as_FloatRegister($src$$reg);
3133 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3134 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3135 %}
3136
3137 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3138 FloatRegister src_reg = as_FloatRegister($src$$reg);
3139 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3140 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3141 %}
3142
3143 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3144 FloatRegister src_reg = as_FloatRegister($src$$reg);
3145 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3146 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3147 %}
3148
3149 // volatile loads and stores
3150
3151 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3152 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3153 rscratch1, stlrb);
3154 %}
3155
3156 enc_class aarch64_enc_stlrb0(memory mem) %{
3157 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3158 rscratch1, stlrb);
3159 %}
3160
3161 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3162 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3163 rscratch1, stlrh);
3164 %}
3165
3166 enc_class aarch64_enc_stlrh0(memory mem) %{
3167 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3168 rscratch1, stlrh);
3169 %}
3170
3171 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3172 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3173 rscratch1, stlrw);
3174 %}
3175
3176 enc_class aarch64_enc_stlrw0(memory mem) %{
3177 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3178 rscratch1, stlrw);
3179 %}
3180
3181 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3182 Register dst_reg = as_Register($dst$$reg);
3183 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3184 rscratch1, ldarb);
3185 __ sxtbw(dst_reg, dst_reg);
3186 %}
3187
3188 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3189 Register dst_reg = as_Register($dst$$reg);
3190 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3191 rscratch1, ldarb);
3192 __ sxtb(dst_reg, dst_reg);
3193 %}
3194
3195 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3196 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3197 rscratch1, ldarb);
3198 %}
3199
3200 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3201 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3202 rscratch1, ldarb);
3203 %}
3204
3205 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3206 Register dst_reg = as_Register($dst$$reg);
3207 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3208 rscratch1, ldarh);
3209 __ sxthw(dst_reg, dst_reg);
3210 %}
3211
3212 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3213 Register dst_reg = as_Register($dst$$reg);
3214 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3215 rscratch1, ldarh);
3216 __ sxth(dst_reg, dst_reg);
3217 %}
3218
3219 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3220 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3221 rscratch1, ldarh);
3222 %}
3223
3224 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3225 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3226 rscratch1, ldarh);
3227 %}
3228
3229 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3230 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3231 rscratch1, ldarw);
3232 %}
3233
3234 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3235 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3236 rscratch1, ldarw);
3237 %}
3238
3239 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3240 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3241 rscratch1, ldar);
3242 %}
3243
3244 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3245 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3246 rscratch1, ldarw);
3247 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3248 %}
3249
3250 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3251 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3252 rscratch1, ldar);
3253 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3254 %}
3255
3256 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3257 Register src_reg = as_Register($src$$reg);
3258 // we sometimes get asked to store the stack pointer into the
3259 // current thread -- we cannot do that directly on AArch64
3260 if (src_reg == r31_sp) {
3261 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3262 __ mov(rscratch2, sp);
3263 src_reg = rscratch2;
3264 }
3265 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3266 rscratch1, stlr);
3267 %}
3268
3269 enc_class aarch64_enc_stlr0(memory mem) %{
3270 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3271 rscratch1, stlr);
3272 %}
3273
3274 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3275 {
3276 FloatRegister src_reg = as_FloatRegister($src$$reg);
3277 __ fmovs(rscratch2, src_reg);
3278 }
3279 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3280 rscratch1, stlrw);
3281 %}
3282
3283 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3284 {
3285 FloatRegister src_reg = as_FloatRegister($src$$reg);
3286 __ fmovd(rscratch2, src_reg);
3287 }
3288 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3289 rscratch1, stlr);
3290 %}
3291
3292 // synchronized read/update encodings
3293
3294 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3295 Register dst_reg = as_Register($dst$$reg);
3296 Register base = as_Register($mem$$base);
3297 int index = $mem$$index;
3298 int scale = $mem$$scale;
3299 int disp = $mem$$disp;
3300 if (index == -1) {
3301 if (disp != 0) {
3302 __ lea(rscratch1, Address(base, disp));
3303 __ ldaxr(dst_reg, rscratch1);
3304 } else {
3305 // TODO
3306 // should we ever get anything other than this case?
3307 __ ldaxr(dst_reg, base);
3308 }
3309 } else {
3310 Register index_reg = as_Register(index);
3311 if (disp == 0) {
3312 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3313 __ ldaxr(dst_reg, rscratch1);
3314 } else {
3315 __ lea(rscratch1, Address(base, disp));
3316 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3317 __ ldaxr(dst_reg, rscratch1);
3318 }
3319 }
3320 %}
3321
3322 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3323 Register src_reg = as_Register($src$$reg);
3324 Register base = as_Register($mem$$base);
3325 int index = $mem$$index;
3326 int scale = $mem$$scale;
3327 int disp = $mem$$disp;
3328 if (index == -1) {
3329 if (disp != 0) {
3330 __ lea(rscratch2, Address(base, disp));
3331 __ stlxr(rscratch1, src_reg, rscratch2);
3332 } else {
3333 // TODO
3334 // should we ever get anything other than this case?
3335 __ stlxr(rscratch1, src_reg, base);
3336 }
3337 } else {
3338 Register index_reg = as_Register(index);
3339 if (disp == 0) {
3340 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3341 __ stlxr(rscratch1, src_reg, rscratch2);
3342 } else {
3343 __ lea(rscratch2, Address(base, disp));
3344 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3345 __ stlxr(rscratch1, src_reg, rscratch2);
3346 }
3347 }
3348 __ cmpw(rscratch1, zr);
3349 %}
3350
3351 // prefetch encodings
3352
3353 enc_class aarch64_enc_prefetchw(memory mem) %{
3354 Register base = as_Register($mem$$base);
3355 int index = $mem$$index;
3356 int scale = $mem$$scale;
3357 int disp = $mem$$disp;
3358 if (index == -1) {
3359 // Fix up any out-of-range offsets.
3360 assert_different_registers(rscratch1, base);
3361 Address addr = Address(base, disp);
3362 addr = __ legitimize_address(addr, 8, rscratch1);
3363 __ prfm(addr, PSTL1KEEP);
3364 } else {
3365 Register index_reg = as_Register(index);
3366 if (disp == 0) {
3367 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3368 } else {
3369 __ lea(rscratch1, Address(base, disp));
3370 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3371 }
3372 }
3373 %}
3374
3375 // mov encodings
3376
3377 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3378 uint32_t con = (uint32_t)$src$$constant;
3379 Register dst_reg = as_Register($dst$$reg);
3380 if (con == 0) {
3381 __ movw(dst_reg, zr);
3382 } else {
3383 __ movw(dst_reg, con);
3384 }
3385 %}
3386
3387 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3388 Register dst_reg = as_Register($dst$$reg);
3389 uint64_t con = (uint64_t)$src$$constant;
3390 if (con == 0) {
3391 __ mov(dst_reg, zr);
3392 } else {
3393 __ mov(dst_reg, con);
3394 }
3395 %}
3396
3397 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3398 Register dst_reg = as_Register($dst$$reg);
3399 address con = (address)$src$$constant;
3400 if (con == nullptr || con == (address)1) {
3401 ShouldNotReachHere();
3402 } else {
3403 relocInfo::relocType rtype = $src->constant_reloc();
3404 if (rtype == relocInfo::oop_type) {
3405 __ movoop(dst_reg, (jobject)con);
3406 } else if (rtype == relocInfo::metadata_type) {
3407 __ mov_metadata(dst_reg, (Metadata*)con);
3408 } else {
3409 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3410 // load fake address constants using a normal move
3411 if (! __ is_valid_AArch64_address(con) ||
3412 con < (address)(uintptr_t)os::vm_page_size() ||
3413 rtype == relocInfo::none) {
3414 __ mov(dst_reg, con);
3415 } else {
3416 // use shorter adrp/add sequence for external_word relocation
3417 uint64_t offset;
3418 __ adrp(dst_reg, Address(con, rtype), offset);
3419 __ add(dst_reg, dst_reg, offset);
3420 }
3421 }
3422 }
3423 %}
3424
3425 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3426 Register dst_reg = as_Register($dst$$reg);
3427 __ mov(dst_reg, zr);
3428 %}
3429
3430 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3431 Register dst_reg = as_Register($dst$$reg);
3432 __ mov(dst_reg, (uint64_t)1);
3433 %}
3434
3435 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3436 Register dst_reg = as_Register($dst$$reg);
3437 address con = (address)$src$$constant;
3438 if (con == nullptr) {
3439 ShouldNotReachHere();
3440 } else {
3441 relocInfo::relocType rtype = $src->constant_reloc();
3442 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3443 __ set_narrow_oop(dst_reg, (jobject)con);
3444 }
3445 %}
3446
3447 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3448 Register dst_reg = as_Register($dst$$reg);
3449 __ mov(dst_reg, zr);
3450 %}
3451
3452 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3453 Register dst_reg = as_Register($dst$$reg);
3454 address con = (address)$src$$constant;
3455 if (con == nullptr) {
3456 ShouldNotReachHere();
3457 } else {
3458 relocInfo::relocType rtype = $src->constant_reloc();
3459 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3460 __ set_narrow_klass(dst_reg, (Klass *)con);
3461 }
3462 %}
3463
3464 // arithmetic encodings
3465
3466 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3467 Register dst_reg = as_Register($dst$$reg);
3468 Register src_reg = as_Register($src1$$reg);
3469 int32_t con = (int32_t)$src2$$constant;
3470 // add has primary == 0, subtract has primary == 1
3471 if ($primary) { con = -con; }
3472 if (con < 0) {
3473 __ subw(dst_reg, src_reg, -con);
3474 } else {
3475 __ addw(dst_reg, src_reg, con);
3476 }
3477 %}
3478
3479 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3480 Register dst_reg = as_Register($dst$$reg);
3481 Register src_reg = as_Register($src1$$reg);
3482 int32_t con = (int32_t)$src2$$constant;
3483 // add has primary == 0, subtract has primary == 1
3484 if ($primary) { con = -con; }
3485 if (con < 0) {
3486 __ sub(dst_reg, src_reg, -con);
3487 } else {
3488 __ add(dst_reg, src_reg, con);
3489 }
3490 %}
3491
3492 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3493 Register dst_reg = as_Register($dst$$reg);
3494 Register src1_reg = as_Register($src1$$reg);
3495 Register src2_reg = as_Register($src2$$reg);
3496 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3497 %}
3498
3499 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3500 Register dst_reg = as_Register($dst$$reg);
3501 Register src1_reg = as_Register($src1$$reg);
3502 Register src2_reg = as_Register($src2$$reg);
3503 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3504 %}
3505
3506 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3507 Register dst_reg = as_Register($dst$$reg);
3508 Register src1_reg = as_Register($src1$$reg);
3509 Register src2_reg = as_Register($src2$$reg);
3510 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3511 %}
3512
3513 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3514 Register dst_reg = as_Register($dst$$reg);
3515 Register src1_reg = as_Register($src1$$reg);
3516 Register src2_reg = as_Register($src2$$reg);
3517 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3518 %}
3519
3520 // compare instruction encodings
3521
3522 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3523 Register reg1 = as_Register($src1$$reg);
3524 Register reg2 = as_Register($src2$$reg);
3525 __ cmpw(reg1, reg2);
3526 %}
3527
3528 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3529 Register reg = as_Register($src1$$reg);
3530 int32_t val = $src2$$constant;
3531 if (val >= 0) {
3532 __ subsw(zr, reg, val);
3533 } else {
3534 __ addsw(zr, reg, -val);
3535 }
3536 %}
3537
3538 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3539 Register reg1 = as_Register($src1$$reg);
3540 uint32_t val = (uint32_t)$src2$$constant;
3541 __ movw(rscratch1, val);
3542 __ cmpw(reg1, rscratch1);
3543 %}
3544
3545 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3546 Register reg1 = as_Register($src1$$reg);
3547 Register reg2 = as_Register($src2$$reg);
3548 __ cmp(reg1, reg2);
3549 %}
3550
3551 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3552 Register reg = as_Register($src1$$reg);
3553 int64_t val = $src2$$constant;
3554 if (val >= 0) {
3555 __ subs(zr, reg, val);
3556 } else if (val != -val) {
3557 __ adds(zr, reg, -val);
3558 } else {
3559 // aargh, Long.MIN_VALUE is a special case
3560 __ orr(rscratch1, zr, (uint64_t)val);
3561 __ subs(zr, reg, rscratch1);
3562 }
3563 %}
3564
3565 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3566 Register reg1 = as_Register($src1$$reg);
3567 uint64_t val = (uint64_t)$src2$$constant;
3568 __ mov(rscratch1, val);
3569 __ cmp(reg1, rscratch1);
3570 %}
3571
3572 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3573 Register reg1 = as_Register($src1$$reg);
3574 Register reg2 = as_Register($src2$$reg);
3575 __ cmp(reg1, reg2);
3576 %}
3577
3578 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3579 Register reg1 = as_Register($src1$$reg);
3580 Register reg2 = as_Register($src2$$reg);
3581 __ cmpw(reg1, reg2);
3582 %}
3583
3584 enc_class aarch64_enc_testp(iRegP src) %{
3585 Register reg = as_Register($src$$reg);
3586 __ cmp(reg, zr);
3587 %}
3588
3589 enc_class aarch64_enc_testn(iRegN src) %{
3590 Register reg = as_Register($src$$reg);
3591 __ cmpw(reg, zr);
3592 %}
3593
3594 enc_class aarch64_enc_b(label lbl) %{
3595 Label *L = $lbl$$label;
3596 __ b(*L);
3597 %}
3598
3599 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3600 Label *L = $lbl$$label;
3601 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3602 %}
3603
3604 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3605 Label *L = $lbl$$label;
3606 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3607 %}
3608
3609 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3610 %{
3611 Register sub_reg = as_Register($sub$$reg);
3612 Register super_reg = as_Register($super$$reg);
3613 Register temp_reg = as_Register($temp$$reg);
3614 Register result_reg = as_Register($result$$reg);
3615
3616 Label miss;
3617 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3618 nullptr, &miss,
3619 /*set_cond_codes:*/ true);
3620 if ($primary) {
3621 __ mov(result_reg, zr);
3622 }
3623 __ bind(miss);
3624 %}
3625
3626 enc_class aarch64_enc_java_static_call(method meth) %{
3627 address addr = (address)$meth$$method;
3628 address call;
3629 if (!_method) {
3630 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3631 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3632 if (call == nullptr) {
3633 ciEnv::current()->record_failure("CodeCache is full");
3634 return;
3635 }
3636 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3637 // The NOP here is purely to ensure that eliding a call to
3638 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3639 __ nop();
3640 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3641 } else {
3642 int method_index = resolved_method_index(masm);
3643 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3644 : static_call_Relocation::spec(method_index);
3645 call = __ trampoline_call(Address(addr, rspec));
3646 if (call == nullptr) {
3647 ciEnv::current()->record_failure("CodeCache is full");
3648 return;
3649 }
3650 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3651 // Calls of the same statically bound method can share
3652 // a stub to the interpreter.
3653 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3654 } else {
3655 // Emit stub for static call
3656 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3657 if (stub == nullptr) {
3658 ciEnv::current()->record_failure("CodeCache is full");
3659 return;
3660 }
3661 }
3662 }
3663
3664 __ post_call_nop();
3665
3666 // Only non uncommon_trap calls need to reinitialize ptrue.
3667 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3668 __ reinitialize_ptrue();
3669 }
3670 %}
3671
3672 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3673 int method_index = resolved_method_index(masm);
3674 address call = __ ic_call((address)$meth$$method, method_index);
3675 if (call == nullptr) {
3676 ciEnv::current()->record_failure("CodeCache is full");
3677 return;
3678 }
3679 __ post_call_nop();
3680 if (Compile::current()->max_vector_size() > 0) {
3681 __ reinitialize_ptrue();
3682 }
3683 %}
3684
3685 enc_class aarch64_enc_call_epilog() %{
3686 if (VerifyStackAtCalls) {
3687 // Check that stack depth is unchanged: find majik cookie on stack
3688 __ call_Unimplemented();
3689 }
3690 %}
3691
3692 enc_class aarch64_enc_java_to_runtime(method meth) %{
3693 // some calls to generated routines (arraycopy code) are scheduled
3694 // by C2 as runtime calls. if so we can call them using a br (they
3695 // will be in a reachable segment) otherwise we have to use a blr
3696 // which loads the absolute address into a register.
3697 address entry = (address)$meth$$method;
3698 CodeBlob *cb = CodeCache::find_blob(entry);
3699 if (cb) {
3700 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3701 if (call == nullptr) {
3702 ciEnv::current()->record_failure("CodeCache is full");
3703 return;
3704 }
3705 __ post_call_nop();
3706 } else {
3707 Label retaddr;
3708 // Make the anchor frame walkable
3709 __ adr(rscratch2, retaddr);
3710 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3711 __ lea(rscratch1, RuntimeAddress(entry));
3712 __ blr(rscratch1);
3713 __ bind(retaddr);
3714 __ post_call_nop();
3715 }
3716 if (Compile::current()->max_vector_size() > 0) {
3717 __ reinitialize_ptrue();
3718 }
3719 %}
3720
3721 enc_class aarch64_enc_rethrow() %{
3722 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3723 %}
3724
3725 enc_class aarch64_enc_ret() %{
3726 #ifdef ASSERT
3727 if (Compile::current()->max_vector_size() > 0) {
3728 __ verify_ptrue();
3729 }
3730 #endif
3731 __ ret(lr);
3732 %}
3733
3734 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3735 Register target_reg = as_Register($jump_target$$reg);
3736 __ br(target_reg);
3737 %}
3738
3739 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3740 Register target_reg = as_Register($jump_target$$reg);
3741 // exception oop should be in r0
3742 // ret addr has been popped into lr
3743 // callee expects it in r3
3744 __ mov(r3, lr);
3745 __ br(target_reg);
3746 %}
3747
3748 %}
3749
3750 //----------FRAME--------------------------------------------------------------
3751 // Definition of frame structure and management information.
3752 //
3753 // S T A C K L A Y O U T Allocators stack-slot number
3754 // | (to get allocators register number
3755 // G Owned by | | v add OptoReg::stack0())
3756 // r CALLER | |
3757 // o | +--------+ pad to even-align allocators stack-slot
3758 // w V | pad0 | numbers; owned by CALLER
3759 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3760 // h ^ | in | 5
3761 // | | args | 4 Holes in incoming args owned by SELF
3762 // | | | | 3
3763 // | | +--------+
3764 // V | | old out| Empty on Intel, window on Sparc
3765 // | old |preserve| Must be even aligned.
3766 // | SP-+--------+----> Matcher::_old_SP, even aligned
3767 // | | in | 3 area for Intel ret address
3768 // Owned by |preserve| Empty on Sparc.
3769 // SELF +--------+
3770 // | | pad2 | 2 pad to align old SP
3771 // | +--------+ 1
3772 // | | locks | 0
3773 // | +--------+----> OptoReg::stack0(), even aligned
3774 // | | pad1 | 11 pad to align new SP
3775 // | +--------+
3776 // | | | 10
3777 // | | spills | 9 spills
3778 // V | | 8 (pad0 slot for callee)
3779 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3780 // ^ | out | 7
3781 // | | args | 6 Holes in outgoing args owned by CALLEE
3782 // Owned by +--------+
3783 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3784 // | new |preserve| Must be even-aligned.
3785 // | SP-+--------+----> Matcher::_new_SP, even aligned
3786 // | | |
3787 //
3788 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3789 // known from SELF's arguments and the Java calling convention.
3790 // Region 6-7 is determined per call site.
3791 // Note 2: If the calling convention leaves holes in the incoming argument
3792 // area, those holes are owned by SELF. Holes in the outgoing area
3793 // are owned by the CALLEE. Holes should not be necessary in the
3794 // incoming area, as the Java calling convention is completely under
3795 // the control of the AD file. Doubles can be sorted and packed to
3796 // avoid holes. Holes in the outgoing arguments may be necessary for
3797 // varargs C calling conventions.
3798 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3799 // even aligned with pad0 as needed.
3800 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3801 // (the latter is true on Intel but is it false on AArch64?)
3802 // region 6-11 is even aligned; it may be padded out more so that
3803 // the region from SP to FP meets the minimum stack alignment.
3804 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3805 // alignment. Region 11, pad1, may be dynamically extended so that
3806 // SP meets the minimum alignment.
3807
3808 frame %{
3809 // These three registers define part of the calling convention
3810 // between compiled code and the interpreter.
3811
3812 // Inline Cache Register or Method for I2C.
3813 inline_cache_reg(R12);
3814
3815 // Number of stack slots consumed by locking an object
3816 sync_stack_slots(2);
3817
3818 // Compiled code's Frame Pointer
3819 frame_pointer(R31);
3820
3821 // Stack alignment requirement
3822 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3823
3824 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3825 // for calls to C. Supports the var-args backing area for register parms.
3826 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3827
3828 // The after-PROLOG location of the return address. Location of
3829 // return address specifies a type (REG or STACK) and a number
3830 // representing the register number (i.e. - use a register name) or
3831 // stack slot.
3832 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3833 // Otherwise, it is above the locks and verification slot and alignment word
3834 // TODO this may well be correct but need to check why that - 2 is there
3835 // ppc port uses 0 but we definitely need to allow for fixed_slots
3836 // which folds in the space used for monitors
3837 return_addr(STACK - 2 +
3838 align_up((Compile::current()->in_preserve_stack_slots() +
3839 Compile::current()->fixed_slots()),
3840 stack_alignment_in_slots()));
3841
3842 // Location of compiled Java return values. Same as C for now.
3843 return_value
3844 %{
3845 // TODO do we allow ideal_reg == Op_RegN???
3846 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3847 "only return normal values");
3848
3849 static const int lo[Op_RegL + 1] = { // enum name
3850 0, // Op_Node
3851 0, // Op_Set
3852 R0_num, // Op_RegN
3853 R0_num, // Op_RegI
3854 R0_num, // Op_RegP
3855 V0_num, // Op_RegF
3856 V0_num, // Op_RegD
3857 R0_num // Op_RegL
3858 };
3859
3860 static const int hi[Op_RegL + 1] = { // enum name
3861 0, // Op_Node
3862 0, // Op_Set
3863 OptoReg::Bad, // Op_RegN
3864 OptoReg::Bad, // Op_RegI
3865 R0_H_num, // Op_RegP
3866 OptoReg::Bad, // Op_RegF
3867 V0_H_num, // Op_RegD
3868 R0_H_num // Op_RegL
3869 };
3870
3871 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3872 %}
3873 %}
3874
3875 //----------ATTRIBUTES---------------------------------------------------------
3876 //----------Operand Attributes-------------------------------------------------
3877 op_attrib op_cost(1); // Required cost attribute
3878
3879 //----------Instruction Attributes---------------------------------------------
3880 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3881 ins_attrib ins_size(32); // Required size attribute (in bits)
3882 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3883 // a non-matching short branch variant
3884 // of some long branch?
3885 ins_attrib ins_alignment(4); // Required alignment attribute (must
3886 // be a power of 2) specifies the
3887 // alignment that some part of the
3888 // instruction (not necessarily the
3889 // start) requires. If > 1, a
3890 // compute_padding() function must be
3891 // provided for the instruction
3892
3893 // Whether this node is expanded during code emission into a sequence of
3894 // instructions and the first instruction can perform an implicit null check.
3895 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3896
3897 //----------OPERANDS-----------------------------------------------------------
3898 // Operand definitions must precede instruction definitions for correct parsing
3899 // in the ADLC because operands constitute user defined types which are used in
3900 // instruction definitions.
3901
3902 //----------Simple Operands----------------------------------------------------
3903
3904 // Integer operands 32 bit
3905 // 32 bit immediate
3906 operand immI()
3907 %{
3908 match(ConI);
3909
3910 op_cost(0);
3911 format %{ %}
3912 interface(CONST_INTER);
3913 %}
3914
3915 // 32 bit zero
3916 operand immI0()
3917 %{
3918 predicate(n->get_int() == 0);
3919 match(ConI);
3920
3921 op_cost(0);
3922 format %{ %}
3923 interface(CONST_INTER);
3924 %}
3925
3926 // 32 bit unit increment
3927 operand immI_1()
3928 %{
3929 predicate(n->get_int() == 1);
3930 match(ConI);
3931
3932 op_cost(0);
3933 format %{ %}
3934 interface(CONST_INTER);
3935 %}
3936
3937 // 32 bit unit decrement
3938 operand immI_M1()
3939 %{
3940 predicate(n->get_int() == -1);
3941 match(ConI);
3942
3943 op_cost(0);
3944 format %{ %}
3945 interface(CONST_INTER);
3946 %}
3947
3948 // Shift values for add/sub extension shift
3949 operand immIExt()
3950 %{
3951 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3952 match(ConI);
3953
3954 op_cost(0);
3955 format %{ %}
3956 interface(CONST_INTER);
3957 %}
3958
3959 operand immI_gt_1()
3960 %{
3961 predicate(n->get_int() > 1);
3962 match(ConI);
3963
3964 op_cost(0);
3965 format %{ %}
3966 interface(CONST_INTER);
3967 %}
3968
3969 operand immI_le_4()
3970 %{
3971 predicate(n->get_int() <= 4);
3972 match(ConI);
3973
3974 op_cost(0);
3975 format %{ %}
3976 interface(CONST_INTER);
3977 %}
3978
3979 operand immI_16()
3980 %{
3981 predicate(n->get_int() == 16);
3982 match(ConI);
3983
3984 op_cost(0);
3985 format %{ %}
3986 interface(CONST_INTER);
3987 %}
3988
3989 operand immI_24()
3990 %{
3991 predicate(n->get_int() == 24);
3992 match(ConI);
3993
3994 op_cost(0);
3995 format %{ %}
3996 interface(CONST_INTER);
3997 %}
3998
3999 operand immI_32()
4000 %{
4001 predicate(n->get_int() == 32);
4002 match(ConI);
4003
4004 op_cost(0);
4005 format %{ %}
4006 interface(CONST_INTER);
4007 %}
4008
4009 operand immI_48()
4010 %{
4011 predicate(n->get_int() == 48);
4012 match(ConI);
4013
4014 op_cost(0);
4015 format %{ %}
4016 interface(CONST_INTER);
4017 %}
4018
4019 operand immI_56()
4020 %{
4021 predicate(n->get_int() == 56);
4022 match(ConI);
4023
4024 op_cost(0);
4025 format %{ %}
4026 interface(CONST_INTER);
4027 %}
4028
4029 operand immI_255()
4030 %{
4031 predicate(n->get_int() == 255);
4032 match(ConI);
4033
4034 op_cost(0);
4035 format %{ %}
4036 interface(CONST_INTER);
4037 %}
4038
4039 operand immI_65535()
4040 %{
4041 predicate(n->get_int() == 65535);
4042 match(ConI);
4043
4044 op_cost(0);
4045 format %{ %}
4046 interface(CONST_INTER);
4047 %}
4048
4049 operand immI_positive()
4050 %{
4051 predicate(n->get_int() > 0);
4052 match(ConI);
4053
4054 op_cost(0);
4055 format %{ %}
4056 interface(CONST_INTER);
4057 %}
4058
4059 // BoolTest condition for signed compare
4060 operand immI_cmp_cond()
4061 %{
4062 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4063 match(ConI);
4064
4065 op_cost(0);
4066 format %{ %}
4067 interface(CONST_INTER);
4068 %}
4069
4070 // BoolTest condition for unsigned compare
4071 operand immI_cmpU_cond()
4072 %{
4073 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4074 match(ConI);
4075
4076 op_cost(0);
4077 format %{ %}
4078 interface(CONST_INTER);
4079 %}
4080
4081 operand immL_255()
4082 %{
4083 predicate(n->get_long() == 255L);
4084 match(ConL);
4085
4086 op_cost(0);
4087 format %{ %}
4088 interface(CONST_INTER);
4089 %}
4090
4091 operand immL_65535()
4092 %{
4093 predicate(n->get_long() == 65535L);
4094 match(ConL);
4095
4096 op_cost(0);
4097 format %{ %}
4098 interface(CONST_INTER);
4099 %}
4100
4101 operand immL_4294967295()
4102 %{
4103 predicate(n->get_long() == 4294967295L);
4104 match(ConL);
4105
4106 op_cost(0);
4107 format %{ %}
4108 interface(CONST_INTER);
4109 %}
4110
4111 operand immL_bitmask()
4112 %{
4113 predicate((n->get_long() != 0)
4114 && ((n->get_long() & 0xc000000000000000l) == 0)
4115 && is_power_of_2(n->get_long() + 1));
4116 match(ConL);
4117
4118 op_cost(0);
4119 format %{ %}
4120 interface(CONST_INTER);
4121 %}
4122
4123 operand immI_bitmask()
4124 %{
4125 predicate((n->get_int() != 0)
4126 && ((n->get_int() & 0xc0000000) == 0)
4127 && is_power_of_2(n->get_int() + 1));
4128 match(ConI);
4129
4130 op_cost(0);
4131 format %{ %}
4132 interface(CONST_INTER);
4133 %}
4134
4135 operand immL_positive_bitmaskI()
4136 %{
4137 predicate((n->get_long() != 0)
4138 && ((julong)n->get_long() < 0x80000000ULL)
4139 && is_power_of_2(n->get_long() + 1));
4140 match(ConL);
4141
4142 op_cost(0);
4143 format %{ %}
4144 interface(CONST_INTER);
4145 %}
4146
4147 // Scale values for scaled offset addressing modes (up to long but not quad)
4148 operand immIScale()
4149 %{
4150 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4151 match(ConI);
4152
4153 op_cost(0);
4154 format %{ %}
4155 interface(CONST_INTER);
4156 %}
4157
4158 // 5 bit signed integer
4159 operand immI5()
4160 %{
4161 predicate(Assembler::is_simm(n->get_int(), 5));
4162 match(ConI);
4163
4164 op_cost(0);
4165 format %{ %}
4166 interface(CONST_INTER);
4167 %}
4168
4169 // 7 bit unsigned integer
4170 operand immIU7()
4171 %{
4172 predicate(Assembler::is_uimm(n->get_int(), 7));
4173 match(ConI);
4174
4175 op_cost(0);
4176 format %{ %}
4177 interface(CONST_INTER);
4178 %}
4179
4180 // Offset for scaled or unscaled immediate loads and stores
4181 operand immIOffset()
4182 %{
4183 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4184 match(ConI);
4185
4186 op_cost(0);
4187 format %{ %}
4188 interface(CONST_INTER);
4189 %}
4190
4191 operand immIOffset1()
4192 %{
4193 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4194 match(ConI);
4195
4196 op_cost(0);
4197 format %{ %}
4198 interface(CONST_INTER);
4199 %}
4200
4201 operand immIOffset2()
4202 %{
4203 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4204 match(ConI);
4205
4206 op_cost(0);
4207 format %{ %}
4208 interface(CONST_INTER);
4209 %}
4210
4211 operand immIOffset4()
4212 %{
4213 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4214 match(ConI);
4215
4216 op_cost(0);
4217 format %{ %}
4218 interface(CONST_INTER);
4219 %}
4220
4221 operand immIOffset8()
4222 %{
4223 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4224 match(ConI);
4225
4226 op_cost(0);
4227 format %{ %}
4228 interface(CONST_INTER);
4229 %}
4230
4231 operand immIOffset16()
4232 %{
4233 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4234 match(ConI);
4235
4236 op_cost(0);
4237 format %{ %}
4238 interface(CONST_INTER);
4239 %}
4240
4241 operand immLOffset()
4242 %{
4243 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4244 match(ConL);
4245
4246 op_cost(0);
4247 format %{ %}
4248 interface(CONST_INTER);
4249 %}
4250
4251 operand immLoffset1()
4252 %{
4253 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4254 match(ConL);
4255
4256 op_cost(0);
4257 format %{ %}
4258 interface(CONST_INTER);
4259 %}
4260
4261 operand immLoffset2()
4262 %{
4263 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4264 match(ConL);
4265
4266 op_cost(0);
4267 format %{ %}
4268 interface(CONST_INTER);
4269 %}
4270
4271 operand immLoffset4()
4272 %{
4273 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4274 match(ConL);
4275
4276 op_cost(0);
4277 format %{ %}
4278 interface(CONST_INTER);
4279 %}
4280
4281 operand immLoffset8()
4282 %{
4283 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4284 match(ConL);
4285
4286 op_cost(0);
4287 format %{ %}
4288 interface(CONST_INTER);
4289 %}
4290
4291 operand immLoffset16()
4292 %{
4293 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4294 match(ConL);
4295
4296 op_cost(0);
4297 format %{ %}
4298 interface(CONST_INTER);
4299 %}
4300
4301 // 5 bit signed long integer
4302 operand immL5()
4303 %{
4304 predicate(Assembler::is_simm(n->get_long(), 5));
4305 match(ConL);
4306
4307 op_cost(0);
4308 format %{ %}
4309 interface(CONST_INTER);
4310 %}
4311
4312 // 7 bit unsigned long integer
4313 operand immLU7()
4314 %{
4315 predicate(Assembler::is_uimm(n->get_long(), 7));
4316 match(ConL);
4317
4318 op_cost(0);
4319 format %{ %}
4320 interface(CONST_INTER);
4321 %}
4322
4323 // 8 bit signed value.
4324 operand immI8()
4325 %{
4326 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4327 match(ConI);
4328
4329 op_cost(0);
4330 format %{ %}
4331 interface(CONST_INTER);
4332 %}
4333
4334 // 8 bit signed value (simm8), or #simm8 LSL 8.
4335 operand immIDupV()
4336 %{
4337 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4338 match(ConI);
4339
4340 op_cost(0);
4341 format %{ %}
4342 interface(CONST_INTER);
4343 %}
4344
4345 // 8 bit signed value (simm8), or #simm8 LSL 8.
4346 operand immLDupV()
4347 %{
4348 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4349 match(ConL);
4350
4351 op_cost(0);
4352 format %{ %}
4353 interface(CONST_INTER);
4354 %}
4355
4356 // 8 bit signed value (simm8), or #simm8 LSL 8.
4357 operand immHDupV()
4358 %{
4359 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4360 match(ConH);
4361
4362 op_cost(0);
4363 format %{ %}
4364 interface(CONST_INTER);
4365 %}
4366
4367 // 8 bit integer valid for vector add sub immediate
4368 operand immBAddSubV()
4369 %{
4370 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4371 match(ConI);
4372
4373 op_cost(0);
4374 format %{ %}
4375 interface(CONST_INTER);
4376 %}
4377
4378 // 32 bit integer valid for add sub immediate
4379 operand immIAddSub()
4380 %{
4381 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4382 match(ConI);
4383 op_cost(0);
4384 format %{ %}
4385 interface(CONST_INTER);
4386 %}
4387
4388 // 32 bit integer valid for vector add sub immediate
4389 operand immIAddSubV()
4390 %{
4391 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4392 match(ConI);
4393
4394 op_cost(0);
4395 format %{ %}
4396 interface(CONST_INTER);
4397 %}
4398
4399 // 32 bit unsigned integer valid for logical immediate
4400
4401 operand immBLog()
4402 %{
4403 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4404 match(ConI);
4405
4406 op_cost(0);
4407 format %{ %}
4408 interface(CONST_INTER);
4409 %}
4410
4411 operand immSLog()
4412 %{
4413 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4414 match(ConI);
4415
4416 op_cost(0);
4417 format %{ %}
4418 interface(CONST_INTER);
4419 %}
4420
4421 operand immILog()
4422 %{
4423 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4424 match(ConI);
4425
4426 op_cost(0);
4427 format %{ %}
4428 interface(CONST_INTER);
4429 %}
4430
4431 // Integer operands 64 bit
4432 // 64 bit immediate
4433 operand immL()
4434 %{
4435 match(ConL);
4436
4437 op_cost(0);
4438 format %{ %}
4439 interface(CONST_INTER);
4440 %}
4441
4442 // 64 bit zero
4443 operand immL0()
4444 %{
4445 predicate(n->get_long() == 0);
4446 match(ConL);
4447
4448 op_cost(0);
4449 format %{ %}
4450 interface(CONST_INTER);
4451 %}
4452
4453 // 64 bit unit decrement
4454 operand immL_M1()
4455 %{
4456 predicate(n->get_long() == -1);
4457 match(ConL);
4458
4459 op_cost(0);
4460 format %{ %}
4461 interface(CONST_INTER);
4462 %}
4463
4464 // 64 bit integer valid for add sub immediate
4465 operand immLAddSub()
4466 %{
4467 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4468 match(ConL);
4469 op_cost(0);
4470 format %{ %}
4471 interface(CONST_INTER);
4472 %}
4473
4474 // 64 bit integer valid for addv subv immediate
4475 operand immLAddSubV()
4476 %{
4477 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4478 match(ConL);
4479
4480 op_cost(0);
4481 format %{ %}
4482 interface(CONST_INTER);
4483 %}
4484
4485 // 64 bit integer valid for logical immediate
4486 operand immLLog()
4487 %{
4488 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4489 match(ConL);
4490 op_cost(0);
4491 format %{ %}
4492 interface(CONST_INTER);
4493 %}
4494
4495 // Long Immediate: low 32-bit mask
4496 operand immL_32bits()
4497 %{
4498 predicate(n->get_long() == 0xFFFFFFFFL);
4499 match(ConL);
4500 op_cost(0);
4501 format %{ %}
4502 interface(CONST_INTER);
4503 %}
4504
4505 // Pointer operands
4506 // Pointer Immediate
4507 operand immP()
4508 %{
4509 match(ConP);
4510
4511 op_cost(0);
4512 format %{ %}
4513 interface(CONST_INTER);
4514 %}
4515
4516 // nullptr Pointer Immediate
4517 operand immP0()
4518 %{
4519 predicate(n->get_ptr() == 0);
4520 match(ConP);
4521
4522 op_cost(0);
4523 format %{ %}
4524 interface(CONST_INTER);
4525 %}
4526
4527 // Pointer Immediate One
4528 // this is used in object initialization (initial object header)
4529 operand immP_1()
4530 %{
4531 predicate(n->get_ptr() == 1);
4532 match(ConP);
4533
4534 op_cost(0);
4535 format %{ %}
4536 interface(CONST_INTER);
4537 %}
4538
4539 // AOT Runtime Constants Address
4540 operand immAOTRuntimeConstantsAddress()
4541 %{
4542 // Check if the address is in the range of AOT Runtime Constants
4543 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4544 match(ConP);
4545
4546 op_cost(0);
4547 format %{ %}
4548 interface(CONST_INTER);
4549 %}
4550
4551 // Float and Double operands
4552 // Double Immediate
4553 operand immD()
4554 %{
4555 match(ConD);
4556 op_cost(0);
4557 format %{ %}
4558 interface(CONST_INTER);
4559 %}
4560
4561 // Double Immediate: +0.0d
4562 operand immD0()
4563 %{
4564 predicate(jlong_cast(n->getd()) == 0);
4565 match(ConD);
4566
4567 op_cost(0);
4568 format %{ %}
4569 interface(CONST_INTER);
4570 %}
4571
4572 // constant 'double +0.0'.
4573 operand immDPacked()
4574 %{
4575 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4576 match(ConD);
4577 op_cost(0);
4578 format %{ %}
4579 interface(CONST_INTER);
4580 %}
4581
4582 // Float Immediate
4583 operand immF()
4584 %{
4585 match(ConF);
4586 op_cost(0);
4587 format %{ %}
4588 interface(CONST_INTER);
4589 %}
4590
4591 // Float Immediate: +0.0f.
4592 operand immF0()
4593 %{
4594 predicate(jint_cast(n->getf()) == 0);
4595 match(ConF);
4596
4597 op_cost(0);
4598 format %{ %}
4599 interface(CONST_INTER);
4600 %}
4601
4602 // Half Float (FP16) Immediate
4603 operand immH()
4604 %{
4605 match(ConH);
4606 op_cost(0);
4607 format %{ %}
4608 interface(CONST_INTER);
4609 %}
4610
4611 //
4612 operand immFPacked()
4613 %{
4614 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4615 match(ConF);
4616 op_cost(0);
4617 format %{ %}
4618 interface(CONST_INTER);
4619 %}
4620
4621 // Narrow pointer operands
4622 // Narrow Pointer Immediate
4623 operand immN()
4624 %{
4625 match(ConN);
4626
4627 op_cost(0);
4628 format %{ %}
4629 interface(CONST_INTER);
4630 %}
4631
4632 // Narrow nullptr Pointer Immediate
4633 operand immN0()
4634 %{
4635 predicate(n->get_narrowcon() == 0);
4636 match(ConN);
4637
4638 op_cost(0);
4639 format %{ %}
4640 interface(CONST_INTER);
4641 %}
4642
4643 operand immNKlass()
4644 %{
4645 match(ConNKlass);
4646
4647 op_cost(0);
4648 format %{ %}
4649 interface(CONST_INTER);
4650 %}
4651
4652 // Integer 32 bit Register Operands
4653 // Integer 32 bitRegister (excludes SP)
4654 operand iRegI()
4655 %{
4656 constraint(ALLOC_IN_RC(any_reg32));
4657 match(RegI);
4658 match(iRegINoSp);
4659 op_cost(0);
4660 format %{ %}
4661 interface(REG_INTER);
4662 %}
4663
4664 // Integer 32 bit Register not Special
4665 operand iRegINoSp()
4666 %{
4667 constraint(ALLOC_IN_RC(no_special_reg32));
4668 match(RegI);
4669 op_cost(0);
4670 format %{ %}
4671 interface(REG_INTER);
4672 %}
4673
4674 // Integer 64 bit Register Operands
4675 // Integer 64 bit Register (includes SP)
4676 operand iRegL()
4677 %{
4678 constraint(ALLOC_IN_RC(any_reg));
4679 match(RegL);
4680 match(iRegLNoSp);
4681 op_cost(0);
4682 format %{ %}
4683 interface(REG_INTER);
4684 %}
4685
4686 // Integer 64 bit Register not Special
4687 operand iRegLNoSp()
4688 %{
4689 constraint(ALLOC_IN_RC(no_special_reg));
4690 match(RegL);
4691 match(iRegL_R0);
4692 format %{ %}
4693 interface(REG_INTER);
4694 %}
4695
4696 // Pointer Register Operands
4697 // Pointer Register
4698 operand iRegP()
4699 %{
4700 constraint(ALLOC_IN_RC(ptr_reg));
4701 match(RegP);
4702 match(iRegPNoSp);
4703 match(iRegP_R0);
4704 //match(iRegP_R2);
4705 //match(iRegP_R4);
4706 match(iRegP_R5);
4707 match(thread_RegP);
4708 op_cost(0);
4709 format %{ %}
4710 interface(REG_INTER);
4711 %}
4712
4713 // Pointer 64 bit Register not Special
4714 operand iRegPNoSp()
4715 %{
4716 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4717 match(RegP);
4718 // match(iRegP);
4719 // match(iRegP_R0);
4720 // match(iRegP_R2);
4721 // match(iRegP_R4);
4722 // match(iRegP_R5);
4723 // match(thread_RegP);
4724 op_cost(0);
4725 format %{ %}
4726 interface(REG_INTER);
4727 %}
4728
4729 // This operand is not allowed to use rfp even if
4730 // rfp is not used to hold the frame pointer.
4731 operand iRegPNoSpNoRfp()
4732 %{
4733 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4734 match(RegP);
4735 match(iRegPNoSp);
4736 op_cost(0);
4737 format %{ %}
4738 interface(REG_INTER);
4739 %}
4740
4741 // Pointer 64 bit Register R0 only
4742 operand iRegP_R0()
4743 %{
4744 constraint(ALLOC_IN_RC(r0_reg));
4745 match(RegP);
4746 // match(iRegP);
4747 match(iRegPNoSp);
4748 op_cost(0);
4749 format %{ %}
4750 interface(REG_INTER);
4751 %}
4752
4753 // Pointer 64 bit Register R1 only
4754 operand iRegP_R1()
4755 %{
4756 constraint(ALLOC_IN_RC(r1_reg));
4757 match(RegP);
4758 // match(iRegP);
4759 match(iRegPNoSp);
4760 op_cost(0);
4761 format %{ %}
4762 interface(REG_INTER);
4763 %}
4764
4765 // Pointer 64 bit Register R2 only
4766 operand iRegP_R2()
4767 %{
4768 constraint(ALLOC_IN_RC(r2_reg));
4769 match(RegP);
4770 // match(iRegP);
4771 match(iRegPNoSp);
4772 op_cost(0);
4773 format %{ %}
4774 interface(REG_INTER);
4775 %}
4776
4777 // Pointer 64 bit Register R3 only
4778 operand iRegP_R3()
4779 %{
4780 constraint(ALLOC_IN_RC(r3_reg));
4781 match(RegP);
4782 // match(iRegP);
4783 match(iRegPNoSp);
4784 op_cost(0);
4785 format %{ %}
4786 interface(REG_INTER);
4787 %}
4788
4789 // Pointer 64 bit Register R4 only
4790 operand iRegP_R4()
4791 %{
4792 constraint(ALLOC_IN_RC(r4_reg));
4793 match(RegP);
4794 // match(iRegP);
4795 match(iRegPNoSp);
4796 op_cost(0);
4797 format %{ %}
4798 interface(REG_INTER);
4799 %}
4800
4801 // Pointer 64 bit Register R5 only
4802 operand iRegP_R5()
4803 %{
4804 constraint(ALLOC_IN_RC(r5_reg));
4805 match(RegP);
4806 // match(iRegP);
4807 match(iRegPNoSp);
4808 op_cost(0);
4809 format %{ %}
4810 interface(REG_INTER);
4811 %}
4812
4813 // Pointer 64 bit Register R10 only
4814 operand iRegP_R10()
4815 %{
4816 constraint(ALLOC_IN_RC(r10_reg));
4817 match(RegP);
4818 // match(iRegP);
4819 match(iRegPNoSp);
4820 op_cost(0);
4821 format %{ %}
4822 interface(REG_INTER);
4823 %}
4824
4825 // Long 64 bit Register R0 only
4826 operand iRegL_R0()
4827 %{
4828 constraint(ALLOC_IN_RC(r0_reg));
4829 match(RegL);
4830 match(iRegLNoSp);
4831 op_cost(0);
4832 format %{ %}
4833 interface(REG_INTER);
4834 %}
4835
4836 // Long 64 bit Register R11 only
4837 operand iRegL_R11()
4838 %{
4839 constraint(ALLOC_IN_RC(r11_reg));
4840 match(RegL);
4841 match(iRegLNoSp);
4842 op_cost(0);
4843 format %{ %}
4844 interface(REG_INTER);
4845 %}
4846
4847 // Register R0 only
4848 operand iRegI_R0()
4849 %{
4850 constraint(ALLOC_IN_RC(int_r0_reg));
4851 match(RegI);
4852 match(iRegINoSp);
4853 op_cost(0);
4854 format %{ %}
4855 interface(REG_INTER);
4856 %}
4857
4858 // Register R2 only
4859 operand iRegI_R2()
4860 %{
4861 constraint(ALLOC_IN_RC(int_r2_reg));
4862 match(RegI);
4863 match(iRegINoSp);
4864 op_cost(0);
4865 format %{ %}
4866 interface(REG_INTER);
4867 %}
4868
4869 // Register R3 only
4870 operand iRegI_R3()
4871 %{
4872 constraint(ALLOC_IN_RC(int_r3_reg));
4873 match(RegI);
4874 match(iRegINoSp);
4875 op_cost(0);
4876 format %{ %}
4877 interface(REG_INTER);
4878 %}
4879
4880
4881 // Register R4 only
4882 operand iRegI_R4()
4883 %{
4884 constraint(ALLOC_IN_RC(int_r4_reg));
4885 match(RegI);
4886 match(iRegINoSp);
4887 op_cost(0);
4888 format %{ %}
4889 interface(REG_INTER);
4890 %}
4891
4892
4893 // Pointer Register Operands
4894 // Narrow Pointer Register
4895 operand iRegN()
4896 %{
4897 constraint(ALLOC_IN_RC(any_reg32));
4898 match(RegN);
4899 match(iRegNNoSp);
4900 op_cost(0);
4901 format %{ %}
4902 interface(REG_INTER);
4903 %}
4904
4905 // Integer 64 bit Register not Special
4906 operand iRegNNoSp()
4907 %{
4908 constraint(ALLOC_IN_RC(no_special_reg32));
4909 match(RegN);
4910 op_cost(0);
4911 format %{ %}
4912 interface(REG_INTER);
4913 %}
4914
4915 // Float Register
4916 // Float register operands
4917 operand vRegF()
4918 %{
4919 constraint(ALLOC_IN_RC(float_reg));
4920 match(RegF);
4921
4922 op_cost(0);
4923 format %{ %}
4924 interface(REG_INTER);
4925 %}
4926
4927 // Double Register
4928 // Double register operands
4929 operand vRegD()
4930 %{
4931 constraint(ALLOC_IN_RC(double_reg));
4932 match(RegD);
4933
4934 op_cost(0);
4935 format %{ %}
4936 interface(REG_INTER);
4937 %}
4938
4939 // Generic vector class. This will be used for
4940 // all vector operands, including NEON and SVE.
4941 operand vReg()
4942 %{
4943 constraint(ALLOC_IN_RC(dynamic));
4944 match(VecA);
4945 match(VecD);
4946 match(VecX);
4947
4948 op_cost(0);
4949 format %{ %}
4950 interface(REG_INTER);
4951 %}
4952
4953 operand vReg_V10()
4954 %{
4955 constraint(ALLOC_IN_RC(v10_veca_reg));
4956 match(vReg);
4957
4958 op_cost(0);
4959 format %{ %}
4960 interface(REG_INTER);
4961 %}
4962
4963 operand vReg_V11()
4964 %{
4965 constraint(ALLOC_IN_RC(v11_veca_reg));
4966 match(vReg);
4967
4968 op_cost(0);
4969 format %{ %}
4970 interface(REG_INTER);
4971 %}
4972
4973 operand vReg_V12()
4974 %{
4975 constraint(ALLOC_IN_RC(v12_veca_reg));
4976 match(vReg);
4977
4978 op_cost(0);
4979 format %{ %}
4980 interface(REG_INTER);
4981 %}
4982
4983 operand vReg_V13()
4984 %{
4985 constraint(ALLOC_IN_RC(v13_veca_reg));
4986 match(vReg);
4987
4988 op_cost(0);
4989 format %{ %}
4990 interface(REG_INTER);
4991 %}
4992
4993 operand vReg_V17()
4994 %{
4995 constraint(ALLOC_IN_RC(v17_veca_reg));
4996 match(vReg);
4997
4998 op_cost(0);
4999 format %{ %}
5000 interface(REG_INTER);
5001 %}
5002
5003 operand vReg_V18()
5004 %{
5005 constraint(ALLOC_IN_RC(v18_veca_reg));
5006 match(vReg);
5007
5008 op_cost(0);
5009 format %{ %}
5010 interface(REG_INTER);
5011 %}
5012
5013 operand vReg_V23()
5014 %{
5015 constraint(ALLOC_IN_RC(v23_veca_reg));
5016 match(vReg);
5017
5018 op_cost(0);
5019 format %{ %}
5020 interface(REG_INTER);
5021 %}
5022
5023 operand vReg_V24()
5024 %{
5025 constraint(ALLOC_IN_RC(v24_veca_reg));
5026 match(vReg);
5027
5028 op_cost(0);
5029 format %{ %}
5030 interface(REG_INTER);
5031 %}
5032
5033 operand vecA()
5034 %{
5035 constraint(ALLOC_IN_RC(vectora_reg));
5036 match(VecA);
5037
5038 op_cost(0);
5039 format %{ %}
5040 interface(REG_INTER);
5041 %}
5042
5043 operand vecD()
5044 %{
5045 constraint(ALLOC_IN_RC(vectord_reg));
5046 match(VecD);
5047
5048 op_cost(0);
5049 format %{ %}
5050 interface(REG_INTER);
5051 %}
5052
5053 operand vecX()
5054 %{
5055 constraint(ALLOC_IN_RC(vectorx_reg));
5056 match(VecX);
5057
5058 op_cost(0);
5059 format %{ %}
5060 interface(REG_INTER);
5061 %}
5062
5063 operand vRegD_V0()
5064 %{
5065 constraint(ALLOC_IN_RC(v0_reg));
5066 match(RegD);
5067 op_cost(0);
5068 format %{ %}
5069 interface(REG_INTER);
5070 %}
5071
5072 operand vRegD_V1()
5073 %{
5074 constraint(ALLOC_IN_RC(v1_reg));
5075 match(RegD);
5076 op_cost(0);
5077 format %{ %}
5078 interface(REG_INTER);
5079 %}
5080
5081 operand vRegD_V2()
5082 %{
5083 constraint(ALLOC_IN_RC(v2_reg));
5084 match(RegD);
5085 op_cost(0);
5086 format %{ %}
5087 interface(REG_INTER);
5088 %}
5089
5090 operand vRegD_V3()
5091 %{
5092 constraint(ALLOC_IN_RC(v3_reg));
5093 match(RegD);
5094 op_cost(0);
5095 format %{ %}
5096 interface(REG_INTER);
5097 %}
5098
5099 operand vRegD_V4()
5100 %{
5101 constraint(ALLOC_IN_RC(v4_reg));
5102 match(RegD);
5103 op_cost(0);
5104 format %{ %}
5105 interface(REG_INTER);
5106 %}
5107
5108 operand vRegD_V5()
5109 %{
5110 constraint(ALLOC_IN_RC(v5_reg));
5111 match(RegD);
5112 op_cost(0);
5113 format %{ %}
5114 interface(REG_INTER);
5115 %}
5116
5117 operand vRegD_V6()
5118 %{
5119 constraint(ALLOC_IN_RC(v6_reg));
5120 match(RegD);
5121 op_cost(0);
5122 format %{ %}
5123 interface(REG_INTER);
5124 %}
5125
5126 operand vRegD_V7()
5127 %{
5128 constraint(ALLOC_IN_RC(v7_reg));
5129 match(RegD);
5130 op_cost(0);
5131 format %{ %}
5132 interface(REG_INTER);
5133 %}
5134
5135 operand vRegD_V12()
5136 %{
5137 constraint(ALLOC_IN_RC(v12_reg));
5138 match(RegD);
5139 op_cost(0);
5140 format %{ %}
5141 interface(REG_INTER);
5142 %}
5143
5144 operand vRegD_V13()
5145 %{
5146 constraint(ALLOC_IN_RC(v13_reg));
5147 match(RegD);
5148 op_cost(0);
5149 format %{ %}
5150 interface(REG_INTER);
5151 %}
5152
5153 operand pReg()
5154 %{
5155 constraint(ALLOC_IN_RC(pr_reg));
5156 match(RegVectMask);
5157 match(pRegGov);
5158 op_cost(0);
5159 format %{ %}
5160 interface(REG_INTER);
5161 %}
5162
5163 operand pRegGov()
5164 %{
5165 constraint(ALLOC_IN_RC(gov_pr));
5166 match(RegVectMask);
5167 match(pReg);
5168 op_cost(0);
5169 format %{ %}
5170 interface(REG_INTER);
5171 %}
5172
5173 operand pRegGov_P0()
5174 %{
5175 constraint(ALLOC_IN_RC(p0_reg));
5176 match(RegVectMask);
5177 op_cost(0);
5178 format %{ %}
5179 interface(REG_INTER);
5180 %}
5181
5182 operand pRegGov_P1()
5183 %{
5184 constraint(ALLOC_IN_RC(p1_reg));
5185 match(RegVectMask);
5186 op_cost(0);
5187 format %{ %}
5188 interface(REG_INTER);
5189 %}
5190
5191 // Flags register, used as output of signed compare instructions
5192
5193 // note that on AArch64 we also use this register as the output for
5194 // for floating point compare instructions (CmpF CmpD). this ensures
5195 // that ordered inequality tests use GT, GE, LT or LE none of which
5196 // pass through cases where the result is unordered i.e. one or both
5197 // inputs to the compare is a NaN. this means that the ideal code can
5198 // replace e.g. a GT with an LE and not end up capturing the NaN case
5199 // (where the comparison should always fail). EQ and NE tests are
5200 // always generated in ideal code so that unordered folds into the NE
5201 // case, matching the behaviour of AArch64 NE.
5202 //
5203 // This differs from x86 where the outputs of FP compares use a
5204 // special FP flags registers and where compares based on this
5205 // register are distinguished into ordered inequalities (cmpOpUCF) and
5206 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5207 // to explicitly handle the unordered case in branches. x86 also has
5208 // to include extra CMoveX rules to accept a cmpOpUCF input.
5209
5210 operand rFlagsReg()
5211 %{
5212 constraint(ALLOC_IN_RC(int_flags));
5213 match(RegFlags);
5214
5215 op_cost(0);
5216 format %{ "RFLAGS" %}
5217 interface(REG_INTER);
5218 %}
5219
5220 // Flags register, used as output of unsigned compare instructions
5221 operand rFlagsRegU()
5222 %{
5223 constraint(ALLOC_IN_RC(int_flags));
5224 match(RegFlags);
5225
5226 op_cost(0);
5227 format %{ "RFLAGSU" %}
5228 interface(REG_INTER);
5229 %}
5230
5231 // Special Registers
5232
5233 // Method Register
5234 operand inline_cache_RegP(iRegP reg)
5235 %{
5236 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5237 match(reg);
5238 match(iRegPNoSp);
5239 op_cost(0);
5240 format %{ %}
5241 interface(REG_INTER);
5242 %}
5243
5244 // Thread Register
5245 operand thread_RegP(iRegP reg)
5246 %{
5247 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5248 match(reg);
5249 op_cost(0);
5250 format %{ %}
5251 interface(REG_INTER);
5252 %}
5253
5254 //----------Memory Operands----------------------------------------------------
5255
5256 operand indirect(iRegP reg)
5257 %{
5258 constraint(ALLOC_IN_RC(ptr_reg));
5259 match(reg);
5260 op_cost(0);
5261 format %{ "[$reg]" %}
5262 interface(MEMORY_INTER) %{
5263 base($reg);
5264 index(0xffffffff);
5265 scale(0x0);
5266 disp(0x0);
5267 %}
5268 %}
5269
5270 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5271 %{
5272 constraint(ALLOC_IN_RC(ptr_reg));
5273 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5274 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5275 op_cost(0);
5276 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5277 interface(MEMORY_INTER) %{
5278 base($reg);
5279 index($ireg);
5280 scale($scale);
5281 disp(0x0);
5282 %}
5283 %}
5284
5285 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5286 %{
5287 constraint(ALLOC_IN_RC(ptr_reg));
5288 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5289 match(AddP reg (LShiftL lreg scale));
5290 op_cost(0);
5291 format %{ "$reg, $lreg lsl($scale)" %}
5292 interface(MEMORY_INTER) %{
5293 base($reg);
5294 index($lreg);
5295 scale($scale);
5296 disp(0x0);
5297 %}
5298 %}
5299
5300 operand indIndexI2L(iRegP reg, iRegI ireg)
5301 %{
5302 constraint(ALLOC_IN_RC(ptr_reg));
5303 match(AddP reg (ConvI2L ireg));
5304 op_cost(0);
5305 format %{ "$reg, $ireg, 0, I2L" %}
5306 interface(MEMORY_INTER) %{
5307 base($reg);
5308 index($ireg);
5309 scale(0x0);
5310 disp(0x0);
5311 %}
5312 %}
5313
5314 operand indIndex(iRegP reg, iRegL lreg)
5315 %{
5316 constraint(ALLOC_IN_RC(ptr_reg));
5317 match(AddP reg lreg);
5318 op_cost(0);
5319 format %{ "$reg, $lreg" %}
5320 interface(MEMORY_INTER) %{
5321 base($reg);
5322 index($lreg);
5323 scale(0x0);
5324 disp(0x0);
5325 %}
5326 %}
5327
5328 operand indOffI1(iRegP reg, immIOffset1 off)
5329 %{
5330 constraint(ALLOC_IN_RC(ptr_reg));
5331 match(AddP reg off);
5332 op_cost(0);
5333 format %{ "[$reg, $off]" %}
5334 interface(MEMORY_INTER) %{
5335 base($reg);
5336 index(0xffffffff);
5337 scale(0x0);
5338 disp($off);
5339 %}
5340 %}
5341
5342 operand indOffI2(iRegP reg, immIOffset2 off)
5343 %{
5344 constraint(ALLOC_IN_RC(ptr_reg));
5345 match(AddP reg off);
5346 op_cost(0);
5347 format %{ "[$reg, $off]" %}
5348 interface(MEMORY_INTER) %{
5349 base($reg);
5350 index(0xffffffff);
5351 scale(0x0);
5352 disp($off);
5353 %}
5354 %}
5355
5356 operand indOffI4(iRegP reg, immIOffset4 off)
5357 %{
5358 constraint(ALLOC_IN_RC(ptr_reg));
5359 match(AddP reg off);
5360 op_cost(0);
5361 format %{ "[$reg, $off]" %}
5362 interface(MEMORY_INTER) %{
5363 base($reg);
5364 index(0xffffffff);
5365 scale(0x0);
5366 disp($off);
5367 %}
5368 %}
5369
5370 operand indOffI8(iRegP reg, immIOffset8 off)
5371 %{
5372 constraint(ALLOC_IN_RC(ptr_reg));
5373 match(AddP reg off);
5374 op_cost(0);
5375 format %{ "[$reg, $off]" %}
5376 interface(MEMORY_INTER) %{
5377 base($reg);
5378 index(0xffffffff);
5379 scale(0x0);
5380 disp($off);
5381 %}
5382 %}
5383
5384 operand indOffI16(iRegP reg, immIOffset16 off)
5385 %{
5386 constraint(ALLOC_IN_RC(ptr_reg));
5387 match(AddP reg off);
5388 op_cost(0);
5389 format %{ "[$reg, $off]" %}
5390 interface(MEMORY_INTER) %{
5391 base($reg);
5392 index(0xffffffff);
5393 scale(0x0);
5394 disp($off);
5395 %}
5396 %}
5397
5398 operand indOffL1(iRegP reg, immLoffset1 off)
5399 %{
5400 constraint(ALLOC_IN_RC(ptr_reg));
5401 match(AddP reg off);
5402 op_cost(0);
5403 format %{ "[$reg, $off]" %}
5404 interface(MEMORY_INTER) %{
5405 base($reg);
5406 index(0xffffffff);
5407 scale(0x0);
5408 disp($off);
5409 %}
5410 %}
5411
5412 operand indOffL2(iRegP reg, immLoffset2 off)
5413 %{
5414 constraint(ALLOC_IN_RC(ptr_reg));
5415 match(AddP reg off);
5416 op_cost(0);
5417 format %{ "[$reg, $off]" %}
5418 interface(MEMORY_INTER) %{
5419 base($reg);
5420 index(0xffffffff);
5421 scale(0x0);
5422 disp($off);
5423 %}
5424 %}
5425
5426 operand indOffL4(iRegP reg, immLoffset4 off)
5427 %{
5428 constraint(ALLOC_IN_RC(ptr_reg));
5429 match(AddP reg off);
5430 op_cost(0);
5431 format %{ "[$reg, $off]" %}
5432 interface(MEMORY_INTER) %{
5433 base($reg);
5434 index(0xffffffff);
5435 scale(0x0);
5436 disp($off);
5437 %}
5438 %}
5439
5440 operand indOffL8(iRegP reg, immLoffset8 off)
5441 %{
5442 constraint(ALLOC_IN_RC(ptr_reg));
5443 match(AddP reg off);
5444 op_cost(0);
5445 format %{ "[$reg, $off]" %}
5446 interface(MEMORY_INTER) %{
5447 base($reg);
5448 index(0xffffffff);
5449 scale(0x0);
5450 disp($off);
5451 %}
5452 %}
5453
5454 operand indOffL16(iRegP reg, immLoffset16 off)
5455 %{
5456 constraint(ALLOC_IN_RC(ptr_reg));
5457 match(AddP reg off);
5458 op_cost(0);
5459 format %{ "[$reg, $off]" %}
5460 interface(MEMORY_INTER) %{
5461 base($reg);
5462 index(0xffffffff);
5463 scale(0x0);
5464 disp($off);
5465 %}
5466 %}
5467
5468 operand indirectX2P(iRegL reg)
5469 %{
5470 constraint(ALLOC_IN_RC(ptr_reg));
5471 match(CastX2P reg);
5472 op_cost(0);
5473 format %{ "[$reg]\t# long -> ptr" %}
5474 interface(MEMORY_INTER) %{
5475 base($reg);
5476 index(0xffffffff);
5477 scale(0x0);
5478 disp(0x0);
5479 %}
5480 %}
5481
5482 operand indOffX2P(iRegL reg, immLOffset off)
5483 %{
5484 constraint(ALLOC_IN_RC(ptr_reg));
5485 match(AddP (CastX2P reg) off);
5486 op_cost(0);
5487 format %{ "[$reg, $off]\t# long -> ptr" %}
5488 interface(MEMORY_INTER) %{
5489 base($reg);
5490 index(0xffffffff);
5491 scale(0x0);
5492 disp($off);
5493 %}
5494 %}
5495
5496 operand indirectN(iRegN reg)
5497 %{
5498 predicate(CompressedOops::shift() == 0);
5499 constraint(ALLOC_IN_RC(ptr_reg));
5500 match(DecodeN reg);
5501 op_cost(0);
5502 format %{ "[$reg]\t# narrow" %}
5503 interface(MEMORY_INTER) %{
5504 base($reg);
5505 index(0xffffffff);
5506 scale(0x0);
5507 disp(0x0);
5508 %}
5509 %}
5510
5511 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5512 %{
5513 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5514 constraint(ALLOC_IN_RC(ptr_reg));
5515 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5516 op_cost(0);
5517 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5518 interface(MEMORY_INTER) %{
5519 base($reg);
5520 index($ireg);
5521 scale($scale);
5522 disp(0x0);
5523 %}
5524 %}
5525
5526 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5527 %{
5528 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5529 constraint(ALLOC_IN_RC(ptr_reg));
5530 match(AddP (DecodeN reg) (LShiftL lreg scale));
5531 op_cost(0);
5532 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5533 interface(MEMORY_INTER) %{
5534 base($reg);
5535 index($lreg);
5536 scale($scale);
5537 disp(0x0);
5538 %}
5539 %}
5540
5541 operand indIndexI2LN(iRegN reg, iRegI ireg)
5542 %{
5543 predicate(CompressedOops::shift() == 0);
5544 constraint(ALLOC_IN_RC(ptr_reg));
5545 match(AddP (DecodeN reg) (ConvI2L ireg));
5546 op_cost(0);
5547 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5548 interface(MEMORY_INTER) %{
5549 base($reg);
5550 index($ireg);
5551 scale(0x0);
5552 disp(0x0);
5553 %}
5554 %}
5555
5556 operand indIndexN(iRegN reg, iRegL lreg)
5557 %{
5558 predicate(CompressedOops::shift() == 0);
5559 constraint(ALLOC_IN_RC(ptr_reg));
5560 match(AddP (DecodeN reg) lreg);
5561 op_cost(0);
5562 format %{ "$reg, $lreg\t# narrow" %}
5563 interface(MEMORY_INTER) %{
5564 base($reg);
5565 index($lreg);
5566 scale(0x0);
5567 disp(0x0);
5568 %}
5569 %}
5570
5571 operand indOffIN(iRegN reg, immIOffset off)
5572 %{
5573 predicate(CompressedOops::shift() == 0);
5574 constraint(ALLOC_IN_RC(ptr_reg));
5575 match(AddP (DecodeN reg) off);
5576 op_cost(0);
5577 format %{ "[$reg, $off]\t# narrow" %}
5578 interface(MEMORY_INTER) %{
5579 base($reg);
5580 index(0xffffffff);
5581 scale(0x0);
5582 disp($off);
5583 %}
5584 %}
5585
5586 operand indOffLN(iRegN reg, immLOffset off)
5587 %{
5588 predicate(CompressedOops::shift() == 0);
5589 constraint(ALLOC_IN_RC(ptr_reg));
5590 match(AddP (DecodeN reg) off);
5591 op_cost(0);
5592 format %{ "[$reg, $off]\t# narrow" %}
5593 interface(MEMORY_INTER) %{
5594 base($reg);
5595 index(0xffffffff);
5596 scale(0x0);
5597 disp($off);
5598 %}
5599 %}
5600
5601
5602 //----------Special Memory Operands--------------------------------------------
5603 // Stack Slot Operand - This operand is used for loading and storing temporary
5604 // values on the stack where a match requires a value to
5605 // flow through memory.
5606 operand stackSlotP(sRegP reg)
5607 %{
5608 constraint(ALLOC_IN_RC(stack_slots));
5609 op_cost(100);
5610 // No match rule because this operand is only generated in matching
5611 // match(RegP);
5612 format %{ "[$reg]" %}
5613 interface(MEMORY_INTER) %{
5614 base(0x1e); // RSP
5615 index(0x0); // No Index
5616 scale(0x0); // No Scale
5617 disp($reg); // Stack Offset
5618 %}
5619 %}
5620
5621 operand stackSlotI(sRegI reg)
5622 %{
5623 constraint(ALLOC_IN_RC(stack_slots));
5624 // No match rule because this operand is only generated in matching
5625 // match(RegI);
5626 format %{ "[$reg]" %}
5627 interface(MEMORY_INTER) %{
5628 base(0x1e); // RSP
5629 index(0x0); // No Index
5630 scale(0x0); // No Scale
5631 disp($reg); // Stack Offset
5632 %}
5633 %}
5634
5635 operand stackSlotF(sRegF reg)
5636 %{
5637 constraint(ALLOC_IN_RC(stack_slots));
5638 // No match rule because this operand is only generated in matching
5639 // match(RegF);
5640 format %{ "[$reg]" %}
5641 interface(MEMORY_INTER) %{
5642 base(0x1e); // RSP
5643 index(0x0); // No Index
5644 scale(0x0); // No Scale
5645 disp($reg); // Stack Offset
5646 %}
5647 %}
5648
5649 operand stackSlotD(sRegD reg)
5650 %{
5651 constraint(ALLOC_IN_RC(stack_slots));
5652 // No match rule because this operand is only generated in matching
5653 // match(RegD);
5654 format %{ "[$reg]" %}
5655 interface(MEMORY_INTER) %{
5656 base(0x1e); // RSP
5657 index(0x0); // No Index
5658 scale(0x0); // No Scale
5659 disp($reg); // Stack Offset
5660 %}
5661 %}
5662
5663 operand stackSlotL(sRegL reg)
5664 %{
5665 constraint(ALLOC_IN_RC(stack_slots));
5666 // No match rule because this operand is only generated in matching
5667 // match(RegL);
5668 format %{ "[$reg]" %}
5669 interface(MEMORY_INTER) %{
5670 base(0x1e); // RSP
5671 index(0x0); // No Index
5672 scale(0x0); // No Scale
5673 disp($reg); // Stack Offset
5674 %}
5675 %}
5676
5677 // Operands for expressing Control Flow
5678 // NOTE: Label is a predefined operand which should not be redefined in
5679 // the AD file. It is generically handled within the ADLC.
5680
5681 //----------Conditional Branch Operands----------------------------------------
5682 // Comparison Op - This is the operation of the comparison, and is limited to
5683 // the following set of codes:
5684 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5685 //
5686 // Other attributes of the comparison, such as unsignedness, are specified
5687 // by the comparison instruction that sets a condition code flags register.
5688 // That result is represented by a flags operand whose subtype is appropriate
5689 // to the unsignedness (etc.) of the comparison.
5690 //
5691 // Later, the instruction which matches both the Comparison Op (a Bool) and
5692 // the flags (produced by the Cmp) specifies the coding of the comparison op
5693 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5694
5695 // used for signed integral comparisons and fp comparisons
5696
5697 operand cmpOp()
5698 %{
5699 match(Bool);
5700
5701 format %{ "" %}
5702 interface(COND_INTER) %{
5703 equal(0x0, "eq");
5704 not_equal(0x1, "ne");
5705 less(0xb, "lt");
5706 greater_equal(0xa, "ge");
5707 less_equal(0xd, "le");
5708 greater(0xc, "gt");
5709 overflow(0x6, "vs");
5710 no_overflow(0x7, "vc");
5711 %}
5712 %}
5713
5714 // used for unsigned integral comparisons
5715
5716 operand cmpOpU()
5717 %{
5718 match(Bool);
5719
5720 format %{ "" %}
5721 interface(COND_INTER) %{
5722 equal(0x0, "eq");
5723 not_equal(0x1, "ne");
5724 less(0x3, "lo");
5725 greater_equal(0x2, "hs");
5726 less_equal(0x9, "ls");
5727 greater(0x8, "hi");
5728 overflow(0x6, "vs");
5729 no_overflow(0x7, "vc");
5730 %}
5731 %}
5732
5733 // used for certain integral comparisons which can be
5734 // converted to cbxx or tbxx instructions
5735
5736 operand cmpOpEqNe()
5737 %{
5738 match(Bool);
5739 op_cost(0);
5740 predicate(n->as_Bool()->_test._test == BoolTest::ne
5741 || n->as_Bool()->_test._test == BoolTest::eq);
5742
5743 format %{ "" %}
5744 interface(COND_INTER) %{
5745 equal(0x0, "eq");
5746 not_equal(0x1, "ne");
5747 less(0xb, "lt");
5748 greater_equal(0xa, "ge");
5749 less_equal(0xd, "le");
5750 greater(0xc, "gt");
5751 overflow(0x6, "vs");
5752 no_overflow(0x7, "vc");
5753 %}
5754 %}
5755
5756 // used for certain integral comparisons which can be
5757 // converted to cbxx or tbxx instructions
5758
5759 operand cmpOpLtGe()
5760 %{
5761 match(Bool);
5762 op_cost(0);
5763
5764 predicate(n->as_Bool()->_test._test == BoolTest::lt
5765 || n->as_Bool()->_test._test == BoolTest::ge);
5766
5767 format %{ "" %}
5768 interface(COND_INTER) %{
5769 equal(0x0, "eq");
5770 not_equal(0x1, "ne");
5771 less(0xb, "lt");
5772 greater_equal(0xa, "ge");
5773 less_equal(0xd, "le");
5774 greater(0xc, "gt");
5775 overflow(0x6, "vs");
5776 no_overflow(0x7, "vc");
5777 %}
5778 %}
5779
5780 // used for certain unsigned integral comparisons which can be
5781 // converted to cbxx or tbxx instructions
5782
5783 operand cmpOpUEqNeLeGt()
5784 %{
5785 match(Bool);
5786 op_cost(0);
5787
5788 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5789 n->as_Bool()->_test._test == BoolTest::ne ||
5790 n->as_Bool()->_test._test == BoolTest::le ||
5791 n->as_Bool()->_test._test == BoolTest::gt);
5792
5793 format %{ "" %}
5794 interface(COND_INTER) %{
5795 equal(0x0, "eq");
5796 not_equal(0x1, "ne");
5797 less(0x3, "lo");
5798 greater_equal(0x2, "hs");
5799 less_equal(0x9, "ls");
5800 greater(0x8, "hi");
5801 overflow(0x6, "vs");
5802 no_overflow(0x7, "vc");
5803 %}
5804 %}
5805
5806 // Special operand allowing long args to int ops to be truncated for free
5807
5808 operand iRegL2I(iRegL reg) %{
5809
5810 op_cost(0);
5811
5812 match(ConvL2I reg);
5813
5814 format %{ "l2i($reg)" %}
5815
5816 interface(REG_INTER)
5817 %}
5818
5819 operand iRegL2P(iRegL reg) %{
5820
5821 op_cost(0);
5822
5823 match(CastX2P reg);
5824
5825 format %{ "l2p($reg)" %}
5826
5827 interface(REG_INTER)
5828 %}
5829
5830 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5831 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5832 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5833 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5834
5835 //----------OPERAND CLASSES----------------------------------------------------
5836 // Operand Classes are groups of operands that are used as to simplify
5837 // instruction definitions by not requiring the AD writer to specify
5838 // separate instructions for every form of operand when the
5839 // instruction accepts multiple operand types with the same basic
5840 // encoding and format. The classic case of this is memory operands.
5841
5842 // memory is used to define read/write location for load/store
5843 // instruction defs. we can turn a memory op into an Address
5844
5845 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5846 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5847
5848 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5849 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5850
5851 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5852 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5853
5854 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5855 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5856
5857 // All of the memory operands. For the pipeline description.
5858 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5859 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5860 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5861
5862
5863 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5864 // operations. it allows the src to be either an iRegI or a (ConvL2I
5865 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5866 // can be elided because the 32-bit instruction will just employ the
5867 // lower 32 bits anyway.
5868 //
5869 // n.b. this does not elide all L2I conversions. if the truncated
5870 // value is consumed by more than one operation then the ConvL2I
5871 // cannot be bundled into the consuming nodes so an l2i gets planted
5872 // (actually a movw $dst $src) and the downstream instructions consume
5873 // the result of the l2i as an iRegI input. That's a shame since the
5874 // movw is actually redundant but its not too costly.
5875
5876 opclass iRegIorL2I(iRegI, iRegL2I);
5877 opclass iRegPorL2P(iRegP, iRegL2P);
5878
5879 //----------PIPELINE-----------------------------------------------------------
5880 // Rules which define the behavior of the target architectures pipeline.
5881
5882 // For specific pipelines, eg A53, define the stages of that pipeline
5883 //pipe_desc(ISS, EX1, EX2, WR);
5884 #define ISS S0
5885 #define EX1 S1
5886 #define EX2 S2
5887 #define WR S3
5888
5889 // Integer ALU reg operation
5890 pipeline %{
5891
5892 attributes %{
5893 // ARM instructions are of fixed length
5894 fixed_size_instructions; // Fixed size instructions TODO does
5895 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5896 // ARM instructions come in 32-bit word units
5897 instruction_unit_size = 4; // An instruction is 4 bytes long
5898 instruction_fetch_unit_size = 64; // The processor fetches one line
5899 instruction_fetch_units = 1; // of 64 bytes
5900 %}
5901
5902 // We don't use an actual pipeline model so don't care about resources
5903 // or description. we do use pipeline classes to introduce fixed
5904 // latencies
5905
5906 //----------RESOURCES----------------------------------------------------------
5907 // Resources are the functional units available to the machine
5908
5909 resources( INS0, INS1, INS01 = INS0 | INS1,
5910 ALU0, ALU1, ALU = ALU0 | ALU1,
5911 MAC,
5912 DIV,
5913 BRANCH,
5914 LDST,
5915 NEON_FP);
5916
5917 //----------PIPELINE DESCRIPTION-----------------------------------------------
5918 // Pipeline Description specifies the stages in the machine's pipeline
5919
5920 // Define the pipeline as a generic 6 stage pipeline
5921 pipe_desc(S0, S1, S2, S3, S4, S5);
5922
5923 //----------PIPELINE CLASSES---------------------------------------------------
5924 // Pipeline Classes describe the stages in which input and output are
5925 // referenced by the hardware pipeline.
5926
5927 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5928 %{
5929 single_instruction;
5930 src1 : S1(read);
5931 src2 : S2(read);
5932 dst : S5(write);
5933 INS01 : ISS;
5934 NEON_FP : S5;
5935 %}
5936
5937 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5938 %{
5939 single_instruction;
5940 src1 : S1(read);
5941 src2 : S2(read);
5942 dst : S5(write);
5943 INS01 : ISS;
5944 NEON_FP : S5;
5945 %}
5946
5947 pipe_class fp_uop_s(vRegF dst, vRegF src)
5948 %{
5949 single_instruction;
5950 src : S1(read);
5951 dst : S5(write);
5952 INS01 : ISS;
5953 NEON_FP : S5;
5954 %}
5955
5956 pipe_class fp_uop_d(vRegD dst, vRegD src)
5957 %{
5958 single_instruction;
5959 src : S1(read);
5960 dst : S5(write);
5961 INS01 : ISS;
5962 NEON_FP : S5;
5963 %}
5964
5965 pipe_class fp_d2f(vRegF dst, vRegD src)
5966 %{
5967 single_instruction;
5968 src : S1(read);
5969 dst : S5(write);
5970 INS01 : ISS;
5971 NEON_FP : S5;
5972 %}
5973
5974 pipe_class fp_f2d(vRegD dst, vRegF src)
5975 %{
5976 single_instruction;
5977 src : S1(read);
5978 dst : S5(write);
5979 INS01 : ISS;
5980 NEON_FP : S5;
5981 %}
5982
5983 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5984 %{
5985 single_instruction;
5986 src : S1(read);
5987 dst : S5(write);
5988 INS01 : ISS;
5989 NEON_FP : S5;
5990 %}
5991
5992 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5993 %{
5994 single_instruction;
5995 src : S1(read);
5996 dst : S5(write);
5997 INS01 : ISS;
5998 NEON_FP : S5;
5999 %}
6000
6001 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6002 %{
6003 single_instruction;
6004 src : S1(read);
6005 dst : S5(write);
6006 INS01 : ISS;
6007 NEON_FP : S5;
6008 %}
6009
6010 pipe_class fp_l2f(vRegF dst, iRegL src)
6011 %{
6012 single_instruction;
6013 src : S1(read);
6014 dst : S5(write);
6015 INS01 : ISS;
6016 NEON_FP : S5;
6017 %}
6018
6019 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6020 %{
6021 single_instruction;
6022 src : S1(read);
6023 dst : S5(write);
6024 INS01 : ISS;
6025 NEON_FP : S5;
6026 %}
6027
6028 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6029 %{
6030 single_instruction;
6031 src : S1(read);
6032 dst : S5(write);
6033 INS01 : ISS;
6034 NEON_FP : S5;
6035 %}
6036
6037 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6038 %{
6039 single_instruction;
6040 src : S1(read);
6041 dst : S5(write);
6042 INS01 : ISS;
6043 NEON_FP : S5;
6044 %}
6045
6046 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6047 %{
6048 single_instruction;
6049 src : S1(read);
6050 dst : S5(write);
6051 INS01 : ISS;
6052 NEON_FP : S5;
6053 %}
6054
6055 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6056 %{
6057 single_instruction;
6058 src1 : S1(read);
6059 src2 : S2(read);
6060 dst : S5(write);
6061 INS0 : ISS;
6062 NEON_FP : S5;
6063 %}
6064
6065 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6066 %{
6067 single_instruction;
6068 src1 : S1(read);
6069 src2 : S2(read);
6070 dst : S5(write);
6071 INS0 : ISS;
6072 NEON_FP : S5;
6073 %}
6074
6075 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6076 %{
6077 single_instruction;
6078 cr : S1(read);
6079 src1 : S1(read);
6080 src2 : S1(read);
6081 dst : S3(write);
6082 INS01 : ISS;
6083 NEON_FP : S3;
6084 %}
6085
6086 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6087 %{
6088 single_instruction;
6089 cr : S1(read);
6090 src1 : S1(read);
6091 src2 : S1(read);
6092 dst : S3(write);
6093 INS01 : ISS;
6094 NEON_FP : S3;
6095 %}
6096
6097 pipe_class fp_imm_s(vRegF dst)
6098 %{
6099 single_instruction;
6100 dst : S3(write);
6101 INS01 : ISS;
6102 NEON_FP : S3;
6103 %}
6104
6105 pipe_class fp_imm_d(vRegD dst)
6106 %{
6107 single_instruction;
6108 dst : S3(write);
6109 INS01 : ISS;
6110 NEON_FP : S3;
6111 %}
6112
6113 pipe_class fp_load_constant_s(vRegF dst)
6114 %{
6115 single_instruction;
6116 dst : S4(write);
6117 INS01 : ISS;
6118 NEON_FP : S4;
6119 %}
6120
6121 pipe_class fp_load_constant_d(vRegD dst)
6122 %{
6123 single_instruction;
6124 dst : S4(write);
6125 INS01 : ISS;
6126 NEON_FP : S4;
6127 %}
6128
6129 //------- Integer ALU operations --------------------------
6130
6131 // Integer ALU reg-reg operation
6132 // Operands needed in EX1, result generated in EX2
6133 // Eg. ADD x0, x1, x2
6134 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6135 %{
6136 single_instruction;
6137 dst : EX2(write);
6138 src1 : EX1(read);
6139 src2 : EX1(read);
6140 INS01 : ISS; // Dual issue as instruction 0 or 1
6141 ALU : EX2;
6142 %}
6143
6144 // Integer ALU reg-reg operation with constant shift
6145 // Shifted register must be available in LATE_ISS instead of EX1
6146 // Eg. ADD x0, x1, x2, LSL #2
6147 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6148 %{
6149 single_instruction;
6150 dst : EX2(write);
6151 src1 : EX1(read);
6152 src2 : ISS(read);
6153 INS01 : ISS;
6154 ALU : EX2;
6155 %}
6156
6157 // Integer ALU reg operation with constant shift
6158 // Eg. LSL x0, x1, #shift
6159 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6160 %{
6161 single_instruction;
6162 dst : EX2(write);
6163 src1 : ISS(read);
6164 INS01 : ISS;
6165 ALU : EX2;
6166 %}
6167
6168 // Integer ALU reg-reg operation with variable shift
6169 // Both operands must be available in LATE_ISS instead of EX1
6170 // Result is available in EX1 instead of EX2
6171 // Eg. LSLV x0, x1, x2
6172 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6173 %{
6174 single_instruction;
6175 dst : EX1(write);
6176 src1 : ISS(read);
6177 src2 : ISS(read);
6178 INS01 : ISS;
6179 ALU : EX1;
6180 %}
6181
6182 // Integer ALU reg-reg operation with extract
6183 // As for _vshift above, but result generated in EX2
6184 // Eg. EXTR x0, x1, x2, #N
6185 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6186 %{
6187 single_instruction;
6188 dst : EX2(write);
6189 src1 : ISS(read);
6190 src2 : ISS(read);
6191 INS1 : ISS; // Can only dual issue as Instruction 1
6192 ALU : EX1;
6193 %}
6194
6195 // Integer ALU reg operation
6196 // Eg. NEG x0, x1
6197 pipe_class ialu_reg(iRegI dst, iRegI src)
6198 %{
6199 single_instruction;
6200 dst : EX2(write);
6201 src : EX1(read);
6202 INS01 : ISS;
6203 ALU : EX2;
6204 %}
6205
6206 // Integer ALU reg mmediate operation
6207 // Eg. ADD x0, x1, #N
6208 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6209 %{
6210 single_instruction;
6211 dst : EX2(write);
6212 src1 : EX1(read);
6213 INS01 : ISS;
6214 ALU : EX2;
6215 %}
6216
6217 // Integer ALU immediate operation (no source operands)
6218 // Eg. MOV x0, #N
6219 pipe_class ialu_imm(iRegI dst)
6220 %{
6221 single_instruction;
6222 dst : EX1(write);
6223 INS01 : ISS;
6224 ALU : EX1;
6225 %}
6226
6227 //------- Compare operation -------------------------------
6228
6229 // Compare reg-reg
6230 // Eg. CMP x0, x1
6231 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6232 %{
6233 single_instruction;
6234 // fixed_latency(16);
6235 cr : EX2(write);
6236 op1 : EX1(read);
6237 op2 : EX1(read);
6238 INS01 : ISS;
6239 ALU : EX2;
6240 %}
6241
6242 // Compare reg-reg
6243 // Eg. CMP x0, #N
6244 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6245 %{
6246 single_instruction;
6247 // fixed_latency(16);
6248 cr : EX2(write);
6249 op1 : EX1(read);
6250 INS01 : ISS;
6251 ALU : EX2;
6252 %}
6253
6254 //------- Conditional instructions ------------------------
6255
6256 // Conditional no operands
6257 // Eg. CSINC x0, zr, zr, <cond>
6258 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6259 %{
6260 single_instruction;
6261 cr : EX1(read);
6262 dst : EX2(write);
6263 INS01 : ISS;
6264 ALU : EX2;
6265 %}
6266
6267 // Conditional 2 operand
6268 // EG. CSEL X0, X1, X2, <cond>
6269 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6270 %{
6271 single_instruction;
6272 cr : EX1(read);
6273 src1 : EX1(read);
6274 src2 : EX1(read);
6275 dst : EX2(write);
6276 INS01 : ISS;
6277 ALU : EX2;
6278 %}
6279
6280 // Conditional 2 operand
6281 // EG. CSEL X0, X1, X2, <cond>
6282 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6283 %{
6284 single_instruction;
6285 cr : EX1(read);
6286 src : EX1(read);
6287 dst : EX2(write);
6288 INS01 : ISS;
6289 ALU : EX2;
6290 %}
6291
6292 //------- Multiply pipeline operations --------------------
6293
6294 // Multiply reg-reg
6295 // Eg. MUL w0, w1, w2
6296 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6297 %{
6298 single_instruction;
6299 dst : WR(write);
6300 src1 : ISS(read);
6301 src2 : ISS(read);
6302 INS01 : ISS;
6303 MAC : WR;
6304 %}
6305
6306 // Multiply accumulate
6307 // Eg. MADD w0, w1, w2, w3
6308 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6309 %{
6310 single_instruction;
6311 dst : WR(write);
6312 src1 : ISS(read);
6313 src2 : ISS(read);
6314 src3 : ISS(read);
6315 INS01 : ISS;
6316 MAC : WR;
6317 %}
6318
6319 // Eg. MUL w0, w1, w2
6320 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6321 %{
6322 single_instruction;
6323 fixed_latency(3); // Maximum latency for 64 bit mul
6324 dst : WR(write);
6325 src1 : ISS(read);
6326 src2 : ISS(read);
6327 INS01 : ISS;
6328 MAC : WR;
6329 %}
6330
6331 // Multiply accumulate
6332 // Eg. MADD w0, w1, w2, w3
6333 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6334 %{
6335 single_instruction;
6336 fixed_latency(3); // Maximum latency for 64 bit mul
6337 dst : WR(write);
6338 src1 : ISS(read);
6339 src2 : ISS(read);
6340 src3 : ISS(read);
6341 INS01 : ISS;
6342 MAC : WR;
6343 %}
6344
6345 //------- Divide pipeline operations --------------------
6346
6347 // Eg. SDIV w0, w1, w2
6348 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6349 %{
6350 single_instruction;
6351 fixed_latency(8); // Maximum latency for 32 bit divide
6352 dst : WR(write);
6353 src1 : ISS(read);
6354 src2 : ISS(read);
6355 INS0 : ISS; // Can only dual issue as instruction 0
6356 DIV : WR;
6357 %}
6358
6359 // Eg. SDIV x0, x1, x2
6360 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6361 %{
6362 single_instruction;
6363 fixed_latency(16); // Maximum latency for 64 bit divide
6364 dst : WR(write);
6365 src1 : ISS(read);
6366 src2 : ISS(read);
6367 INS0 : ISS; // Can only dual issue as instruction 0
6368 DIV : WR;
6369 %}
6370
6371 //------- Load pipeline operations ------------------------
6372
6373 // Load - prefetch
6374 // Eg. PFRM <mem>
6375 pipe_class iload_prefetch(memory mem)
6376 %{
6377 single_instruction;
6378 mem : ISS(read);
6379 INS01 : ISS;
6380 LDST : WR;
6381 %}
6382
6383 // Load - reg, mem
6384 // Eg. LDR x0, <mem>
6385 pipe_class iload_reg_mem(iRegI dst, memory mem)
6386 %{
6387 single_instruction;
6388 dst : WR(write);
6389 mem : ISS(read);
6390 INS01 : ISS;
6391 LDST : WR;
6392 %}
6393
6394 // Load - reg, reg
6395 // Eg. LDR x0, [sp, x1]
6396 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6397 %{
6398 single_instruction;
6399 dst : WR(write);
6400 src : ISS(read);
6401 INS01 : ISS;
6402 LDST : WR;
6403 %}
6404
6405 //------- Store pipeline operations -----------------------
6406
6407 // Store - zr, mem
6408 // Eg. STR zr, <mem>
6409 pipe_class istore_mem(memory mem)
6410 %{
6411 single_instruction;
6412 mem : ISS(read);
6413 INS01 : ISS;
6414 LDST : WR;
6415 %}
6416
6417 // Store - reg, mem
6418 // Eg. STR x0, <mem>
6419 pipe_class istore_reg_mem(iRegI src, memory mem)
6420 %{
6421 single_instruction;
6422 mem : ISS(read);
6423 src : EX2(read);
6424 INS01 : ISS;
6425 LDST : WR;
6426 %}
6427
6428 // Store - reg, reg
6429 // Eg. STR x0, [sp, x1]
6430 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6431 %{
6432 single_instruction;
6433 dst : ISS(read);
6434 src : EX2(read);
6435 INS01 : ISS;
6436 LDST : WR;
6437 %}
6438
6439 //------- Store pipeline operations -----------------------
6440
6441 // Branch
6442 pipe_class pipe_branch()
6443 %{
6444 single_instruction;
6445 INS01 : ISS;
6446 BRANCH : EX1;
6447 %}
6448
6449 // Conditional branch
6450 pipe_class pipe_branch_cond(rFlagsReg cr)
6451 %{
6452 single_instruction;
6453 cr : EX1(read);
6454 INS01 : ISS;
6455 BRANCH : EX1;
6456 %}
6457
6458 // Compare & Branch
6459 // EG. CBZ/CBNZ
6460 pipe_class pipe_cmp_branch(iRegI op1)
6461 %{
6462 single_instruction;
6463 op1 : EX1(read);
6464 INS01 : ISS;
6465 BRANCH : EX1;
6466 %}
6467
6468 //------- Synchronisation operations ----------------------
6469
6470 // Any operation requiring serialization.
6471 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6472 pipe_class pipe_serial()
6473 %{
6474 single_instruction;
6475 force_serialization;
6476 fixed_latency(16);
6477 INS01 : ISS(2); // Cannot dual issue with any other instruction
6478 LDST : WR;
6479 %}
6480
6481 // Generic big/slow expanded idiom - also serialized
6482 pipe_class pipe_slow()
6483 %{
6484 instruction_count(10);
6485 multiple_bundles;
6486 force_serialization;
6487 fixed_latency(16);
6488 INS01 : ISS(2); // Cannot dual issue with any other instruction
6489 LDST : WR;
6490 %}
6491
6492 // Empty pipeline class
6493 pipe_class pipe_class_empty()
6494 %{
6495 single_instruction;
6496 fixed_latency(0);
6497 %}
6498
6499 // Default pipeline class.
6500 pipe_class pipe_class_default()
6501 %{
6502 single_instruction;
6503 fixed_latency(2);
6504 %}
6505
6506 // Pipeline class for compares.
6507 pipe_class pipe_class_compare()
6508 %{
6509 single_instruction;
6510 fixed_latency(16);
6511 %}
6512
6513 // Pipeline class for memory operations.
6514 pipe_class pipe_class_memory()
6515 %{
6516 single_instruction;
6517 fixed_latency(16);
6518 %}
6519
6520 // Pipeline class for call.
6521 pipe_class pipe_class_call()
6522 %{
6523 single_instruction;
6524 fixed_latency(100);
6525 %}
6526
6527 // Define the class for the Nop node.
6528 define %{
6529 MachNop = pipe_class_empty;
6530 %}
6531
6532 %}
6533 //----------INSTRUCTIONS-------------------------------------------------------
6534 //
6535 // match -- States which machine-independent subtree may be replaced
6536 // by this instruction.
6537 // ins_cost -- The estimated cost of this instruction is used by instruction
6538 // selection to identify a minimum cost tree of machine
6539 // instructions that matches a tree of machine-independent
6540 // instructions.
6541 // format -- A string providing the disassembly for this instruction.
6542 // The value of an instruction's operand may be inserted
6543 // by referring to it with a '$' prefix.
6544 // opcode -- Three instruction opcodes may be provided. These are referred
6545 // to within an encode class as $primary, $secondary, and $tertiary
6546 // rrspectively. The primary opcode is commonly used to
6547 // indicate the type of machine instruction, while secondary
6548 // and tertiary are often used for prefix options or addressing
6549 // modes.
6550 // ins_encode -- A list of encode classes with parameters. The encode class
6551 // name must have been defined in an 'enc_class' specification
6552 // in the encode section of the architecture description.
6553
6554 // ============================================================================
6555 // Memory (Load/Store) Instructions
6556
6557 // Load Instructions
6558
6559 // Load Byte (8 bit signed)
6560 instruct loadB(iRegINoSp dst, memory1 mem)
6561 %{
6562 match(Set dst (LoadB mem));
6563 predicate(!needs_acquiring_load(n));
6564
6565 ins_cost(4 * INSN_COST);
6566 format %{ "ldrsbw $dst, $mem\t# byte" %}
6567
6568 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6569
6570 ins_pipe(iload_reg_mem);
6571 %}
6572
6573 // Load Byte (8 bit signed) into long
6574 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6575 %{
6576 match(Set dst (ConvI2L (LoadB mem)));
6577 predicate(!needs_acquiring_load(n->in(1)));
6578
6579 ins_cost(4 * INSN_COST);
6580 format %{ "ldrsb $dst, $mem\t# byte" %}
6581
6582 ins_encode(aarch64_enc_ldrsb(dst, mem));
6583
6584 ins_pipe(iload_reg_mem);
6585 %}
6586
6587 // Load Byte (8 bit unsigned)
6588 instruct loadUB(iRegINoSp dst, memory1 mem)
6589 %{
6590 match(Set dst (LoadUB mem));
6591 predicate(!needs_acquiring_load(n));
6592
6593 ins_cost(4 * INSN_COST);
6594 format %{ "ldrbw $dst, $mem\t# byte" %}
6595
6596 ins_encode(aarch64_enc_ldrb(dst, mem));
6597
6598 ins_pipe(iload_reg_mem);
6599 %}
6600
6601 // Load Byte (8 bit unsigned) into long
6602 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6603 %{
6604 match(Set dst (ConvI2L (LoadUB mem)));
6605 predicate(!needs_acquiring_load(n->in(1)));
6606
6607 ins_cost(4 * INSN_COST);
6608 format %{ "ldrb $dst, $mem\t# byte" %}
6609
6610 ins_encode(aarch64_enc_ldrb(dst, mem));
6611
6612 ins_pipe(iload_reg_mem);
6613 %}
6614
6615 // Load Short (16 bit signed)
6616 instruct loadS(iRegINoSp dst, memory2 mem)
6617 %{
6618 match(Set dst (LoadS mem));
6619 predicate(!needs_acquiring_load(n));
6620
6621 ins_cost(4 * INSN_COST);
6622 format %{ "ldrshw $dst, $mem\t# short" %}
6623
6624 ins_encode(aarch64_enc_ldrshw(dst, mem));
6625
6626 ins_pipe(iload_reg_mem);
6627 %}
6628
6629 // Load Short (16 bit signed) into long
6630 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6631 %{
6632 match(Set dst (ConvI2L (LoadS mem)));
6633 predicate(!needs_acquiring_load(n->in(1)));
6634
6635 ins_cost(4 * INSN_COST);
6636 format %{ "ldrsh $dst, $mem\t# short" %}
6637
6638 ins_encode(aarch64_enc_ldrsh(dst, mem));
6639
6640 ins_pipe(iload_reg_mem);
6641 %}
6642
6643 // Load Char (16 bit unsigned)
6644 instruct loadUS(iRegINoSp dst, memory2 mem)
6645 %{
6646 match(Set dst (LoadUS mem));
6647 predicate(!needs_acquiring_load(n));
6648
6649 ins_cost(4 * INSN_COST);
6650 format %{ "ldrh $dst, $mem\t# short" %}
6651
6652 ins_encode(aarch64_enc_ldrh(dst, mem));
6653
6654 ins_pipe(iload_reg_mem);
6655 %}
6656
6657 // Load Short/Char (16 bit unsigned) into long
6658 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6659 %{
6660 match(Set dst (ConvI2L (LoadUS mem)));
6661 predicate(!needs_acquiring_load(n->in(1)));
6662
6663 ins_cost(4 * INSN_COST);
6664 format %{ "ldrh $dst, $mem\t# short" %}
6665
6666 ins_encode(aarch64_enc_ldrh(dst, mem));
6667
6668 ins_pipe(iload_reg_mem);
6669 %}
6670
6671 // Load Integer (32 bit signed)
6672 instruct loadI(iRegINoSp dst, memory4 mem)
6673 %{
6674 match(Set dst (LoadI mem));
6675 predicate(!needs_acquiring_load(n));
6676
6677 ins_cost(4 * INSN_COST);
6678 format %{ "ldrw $dst, $mem\t# int" %}
6679
6680 ins_encode(aarch64_enc_ldrw(dst, mem));
6681
6682 ins_pipe(iload_reg_mem);
6683 %}
6684
6685 // Load Integer (32 bit signed) into long
6686 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6687 %{
6688 match(Set dst (ConvI2L (LoadI mem)));
6689 predicate(!needs_acquiring_load(n->in(1)));
6690
6691 ins_cost(4 * INSN_COST);
6692 format %{ "ldrsw $dst, $mem\t# int" %}
6693
6694 ins_encode(aarch64_enc_ldrsw(dst, mem));
6695
6696 ins_pipe(iload_reg_mem);
6697 %}
6698
6699 // Load Integer (32 bit unsigned) into long
6700 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6701 %{
6702 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6703 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6704
6705 ins_cost(4 * INSN_COST);
6706 format %{ "ldrw $dst, $mem\t# int" %}
6707
6708 ins_encode(aarch64_enc_ldrw(dst, mem));
6709
6710 ins_pipe(iload_reg_mem);
6711 %}
6712
6713 // Load Long (64 bit signed)
6714 instruct loadL(iRegLNoSp dst, memory8 mem)
6715 %{
6716 match(Set dst (LoadL mem));
6717 predicate(!needs_acquiring_load(n));
6718
6719 ins_cost(4 * INSN_COST);
6720 format %{ "ldr $dst, $mem\t# int" %}
6721
6722 ins_encode(aarch64_enc_ldr(dst, mem));
6723
6724 ins_pipe(iload_reg_mem);
6725 %}
6726
6727 // Load Range
6728 instruct loadRange(iRegINoSp dst, memory4 mem)
6729 %{
6730 match(Set dst (LoadRange mem));
6731
6732 ins_cost(4 * INSN_COST);
6733 format %{ "ldrw $dst, $mem\t# range" %}
6734
6735 ins_encode(aarch64_enc_ldrw(dst, mem));
6736
6737 ins_pipe(iload_reg_mem);
6738 %}
6739
6740 // Load Pointer
6741 instruct loadP(iRegPNoSp dst, memory8 mem)
6742 %{
6743 match(Set dst (LoadP mem));
6744 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6745
6746 ins_cost(4 * INSN_COST);
6747 format %{ "ldr $dst, $mem\t# ptr" %}
6748
6749 ins_encode(aarch64_enc_ldr(dst, mem));
6750
6751 ins_pipe(iload_reg_mem);
6752 %}
6753
6754 // Load Compressed Pointer
6755 instruct loadN(iRegNNoSp dst, memory4 mem)
6756 %{
6757 match(Set dst (LoadN mem));
6758 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6759
6760 ins_cost(4 * INSN_COST);
6761 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6762
6763 ins_encode(aarch64_enc_ldrw(dst, mem));
6764
6765 ins_pipe(iload_reg_mem);
6766 %}
6767
6768 // Load Klass Pointer
6769 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6770 %{
6771 match(Set dst (LoadKlass mem));
6772 predicate(!needs_acquiring_load(n));
6773
6774 ins_cost(4 * INSN_COST);
6775 format %{ "ldr $dst, $mem\t# class" %}
6776
6777 ins_encode(aarch64_enc_ldr(dst, mem));
6778
6779 ins_pipe(iload_reg_mem);
6780 %}
6781
6782 // Load Narrow Klass Pointer
6783 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6784 %{
6785 match(Set dst (LoadNKlass mem));
6786 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6787
6788 ins_cost(4 * INSN_COST);
6789 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6790
6791 ins_encode(aarch64_enc_ldrw(dst, mem));
6792
6793 ins_pipe(iload_reg_mem);
6794 %}
6795
6796 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6797 %{
6798 match(Set dst (LoadNKlass mem));
6799 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6800
6801 ins_cost(4 * INSN_COST);
6802 format %{
6803 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6804 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6805 %}
6806 ins_encode %{
6807 // inlined aarch64_enc_ldrw
6808 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6809 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6810 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6811 %}
6812 ins_pipe(iload_reg_mem);
6813 %}
6814
6815 // Load Float
6816 instruct loadF(vRegF dst, memory4 mem)
6817 %{
6818 match(Set dst (LoadF mem));
6819 predicate(!needs_acquiring_load(n));
6820
6821 ins_cost(4 * INSN_COST);
6822 format %{ "ldrs $dst, $mem\t# float" %}
6823
6824 ins_encode( aarch64_enc_ldrs(dst, mem) );
6825
6826 ins_pipe(pipe_class_memory);
6827 %}
6828
6829 // Load Double
6830 instruct loadD(vRegD dst, memory8 mem)
6831 %{
6832 match(Set dst (LoadD mem));
6833 predicate(!needs_acquiring_load(n));
6834
6835 ins_cost(4 * INSN_COST);
6836 format %{ "ldrd $dst, $mem\t# double" %}
6837
6838 ins_encode( aarch64_enc_ldrd(dst, mem) );
6839
6840 ins_pipe(pipe_class_memory);
6841 %}
6842
6843
6844 // Load Int Constant
6845 instruct loadConI(iRegINoSp dst, immI src)
6846 %{
6847 match(Set dst src);
6848
6849 ins_cost(INSN_COST);
6850 format %{ "mov $dst, $src\t# int" %}
6851
6852 ins_encode( aarch64_enc_movw_imm(dst, src) );
6853
6854 ins_pipe(ialu_imm);
6855 %}
6856
6857 // Load Long Constant
6858 instruct loadConL(iRegLNoSp dst, immL src)
6859 %{
6860 match(Set dst src);
6861
6862 ins_cost(INSN_COST);
6863 format %{ "mov $dst, $src\t# long" %}
6864
6865 ins_encode( aarch64_enc_mov_imm(dst, src) );
6866
6867 ins_pipe(ialu_imm);
6868 %}
6869
6870 // Load Pointer Constant
6871
6872 instruct loadConP(iRegPNoSp dst, immP con)
6873 %{
6874 match(Set dst con);
6875
6876 ins_cost(INSN_COST * 4);
6877 format %{
6878 "mov $dst, $con\t# ptr\n\t"
6879 %}
6880
6881 ins_encode(aarch64_enc_mov_p(dst, con));
6882
6883 ins_pipe(ialu_imm);
6884 %}
6885
6886 // Load Null Pointer Constant
6887
6888 instruct loadConP0(iRegPNoSp dst, immP0 con)
6889 %{
6890 match(Set dst con);
6891
6892 ins_cost(INSN_COST);
6893 format %{ "mov $dst, $con\t# nullptr ptr" %}
6894
6895 ins_encode(aarch64_enc_mov_p0(dst, con));
6896
6897 ins_pipe(ialu_imm);
6898 %}
6899
6900 // Load Pointer Constant One
6901
6902 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6903 %{
6904 match(Set dst con);
6905
6906 ins_cost(INSN_COST);
6907 format %{ "mov $dst, $con\t# nullptr ptr" %}
6908
6909 ins_encode(aarch64_enc_mov_p1(dst, con));
6910
6911 ins_pipe(ialu_imm);
6912 %}
6913
6914 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6915 %{
6916 match(Set dst con);
6917
6918 ins_cost(INSN_COST);
6919 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6920
6921 ins_encode %{
6922 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6923 %}
6924
6925 ins_pipe(ialu_imm);
6926 %}
6927
6928 // Load Narrow Pointer Constant
6929
6930 instruct loadConN(iRegNNoSp dst, immN con)
6931 %{
6932 match(Set dst con);
6933
6934 ins_cost(INSN_COST * 4);
6935 format %{ "mov $dst, $con\t# compressed ptr" %}
6936
6937 ins_encode(aarch64_enc_mov_n(dst, con));
6938
6939 ins_pipe(ialu_imm);
6940 %}
6941
6942 // Load Narrow Null Pointer Constant
6943
6944 instruct loadConN0(iRegNNoSp dst, immN0 con)
6945 %{
6946 match(Set dst con);
6947
6948 ins_cost(INSN_COST);
6949 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6950
6951 ins_encode(aarch64_enc_mov_n0(dst, con));
6952
6953 ins_pipe(ialu_imm);
6954 %}
6955
6956 // Load Narrow Klass Constant
6957
6958 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6959 %{
6960 match(Set dst con);
6961
6962 ins_cost(INSN_COST);
6963 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6964
6965 ins_encode(aarch64_enc_mov_nk(dst, con));
6966
6967 ins_pipe(ialu_imm);
6968 %}
6969
6970 // Load Packed Float Constant
6971
6972 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6973 match(Set dst con);
6974 ins_cost(INSN_COST * 4);
6975 format %{ "fmovs $dst, $con"%}
6976 ins_encode %{
6977 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6978 %}
6979
6980 ins_pipe(fp_imm_s);
6981 %}
6982
6983 // Load Float Constant
6984
6985 instruct loadConF(vRegF dst, immF con) %{
6986 match(Set dst con);
6987
6988 ins_cost(INSN_COST * 4);
6989
6990 format %{
6991 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6992 %}
6993
6994 ins_encode %{
6995 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6996 %}
6997
6998 ins_pipe(fp_load_constant_s);
6999 %}
7000
7001 // Load Packed Double Constant
7002
7003 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7004 match(Set dst con);
7005 ins_cost(INSN_COST);
7006 format %{ "fmovd $dst, $con"%}
7007 ins_encode %{
7008 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7009 %}
7010
7011 ins_pipe(fp_imm_d);
7012 %}
7013
7014 // Load Double Constant
7015
7016 instruct loadConD(vRegD dst, immD con) %{
7017 match(Set dst con);
7018
7019 ins_cost(INSN_COST * 5);
7020 format %{
7021 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7022 %}
7023
7024 ins_encode %{
7025 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7026 %}
7027
7028 ins_pipe(fp_load_constant_d);
7029 %}
7030
7031 // Load Half Float Constant
7032 instruct loadConH(vRegF dst, immH con) %{
7033 match(Set dst con);
7034 format %{ "mov rscratch1, $con\n\t"
7035 "fmov $dst, rscratch1"
7036 %}
7037 ins_encode %{
7038 __ movw(rscratch1, (uint32_t)$con$$constant);
7039 __ fmovs($dst$$FloatRegister, rscratch1);
7040 %}
7041 ins_pipe(pipe_class_default);
7042 %}
7043
7044 // Store Instructions
7045
7046 // Store Byte
7047 instruct storeB(iRegIorL2I src, memory1 mem)
7048 %{
7049 match(Set mem (StoreB mem src));
7050 predicate(!needs_releasing_store(n));
7051
7052 ins_cost(INSN_COST);
7053 format %{ "strb $src, $mem\t# byte" %}
7054
7055 ins_encode(aarch64_enc_strb(src, mem));
7056
7057 ins_pipe(istore_reg_mem);
7058 %}
7059
7060
7061 instruct storeimmB0(immI0 zero, memory1 mem)
7062 %{
7063 match(Set mem (StoreB mem zero));
7064 predicate(!needs_releasing_store(n));
7065
7066 ins_cost(INSN_COST);
7067 format %{ "strb rscractch2, $mem\t# byte" %}
7068
7069 ins_encode(aarch64_enc_strb0(mem));
7070
7071 ins_pipe(istore_mem);
7072 %}
7073
7074 // Store Char/Short
7075 instruct storeC(iRegIorL2I src, memory2 mem)
7076 %{
7077 match(Set mem (StoreC mem src));
7078 predicate(!needs_releasing_store(n));
7079
7080 ins_cost(INSN_COST);
7081 format %{ "strh $src, $mem\t# short" %}
7082
7083 ins_encode(aarch64_enc_strh(src, mem));
7084
7085 ins_pipe(istore_reg_mem);
7086 %}
7087
7088 instruct storeimmC0(immI0 zero, memory2 mem)
7089 %{
7090 match(Set mem (StoreC mem zero));
7091 predicate(!needs_releasing_store(n));
7092
7093 ins_cost(INSN_COST);
7094 format %{ "strh zr, $mem\t# short" %}
7095
7096 ins_encode(aarch64_enc_strh0(mem));
7097
7098 ins_pipe(istore_mem);
7099 %}
7100
7101 // Store Integer
7102
7103 instruct storeI(iRegIorL2I src, memory4 mem)
7104 %{
7105 match(Set mem(StoreI mem src));
7106 predicate(!needs_releasing_store(n));
7107
7108 ins_cost(INSN_COST);
7109 format %{ "strw $src, $mem\t# int" %}
7110
7111 ins_encode(aarch64_enc_strw(src, mem));
7112
7113 ins_pipe(istore_reg_mem);
7114 %}
7115
7116 instruct storeimmI0(immI0 zero, memory4 mem)
7117 %{
7118 match(Set mem(StoreI mem zero));
7119 predicate(!needs_releasing_store(n));
7120
7121 ins_cost(INSN_COST);
7122 format %{ "strw zr, $mem\t# int" %}
7123
7124 ins_encode(aarch64_enc_strw0(mem));
7125
7126 ins_pipe(istore_mem);
7127 %}
7128
7129 // Store Long (64 bit signed)
7130 instruct storeL(iRegL src, memory8 mem)
7131 %{
7132 match(Set mem (StoreL mem src));
7133 predicate(!needs_releasing_store(n));
7134
7135 ins_cost(INSN_COST);
7136 format %{ "str $src, $mem\t# int" %}
7137
7138 ins_encode(aarch64_enc_str(src, mem));
7139
7140 ins_pipe(istore_reg_mem);
7141 %}
7142
7143 // Store Long (64 bit signed)
7144 instruct storeimmL0(immL0 zero, memory8 mem)
7145 %{
7146 match(Set mem (StoreL mem zero));
7147 predicate(!needs_releasing_store(n));
7148
7149 ins_cost(INSN_COST);
7150 format %{ "str zr, $mem\t# int" %}
7151
7152 ins_encode(aarch64_enc_str0(mem));
7153
7154 ins_pipe(istore_mem);
7155 %}
7156
7157 // Store Pointer
7158 instruct storeP(iRegP src, memory8 mem)
7159 %{
7160 match(Set mem (StoreP mem src));
7161 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7162
7163 ins_cost(INSN_COST);
7164 format %{ "str $src, $mem\t# ptr" %}
7165
7166 ins_encode(aarch64_enc_str(src, mem));
7167
7168 ins_pipe(istore_reg_mem);
7169 %}
7170
7171 // Store Pointer
7172 instruct storeimmP0(immP0 zero, memory8 mem)
7173 %{
7174 match(Set mem (StoreP mem zero));
7175 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7176
7177 ins_cost(INSN_COST);
7178 format %{ "str zr, $mem\t# ptr" %}
7179
7180 ins_encode(aarch64_enc_str0(mem));
7181
7182 ins_pipe(istore_mem);
7183 %}
7184
7185 // Store Compressed Pointer
7186 instruct storeN(iRegN src, memory4 mem)
7187 %{
7188 match(Set mem (StoreN mem src));
7189 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7190
7191 ins_cost(INSN_COST);
7192 format %{ "strw $src, $mem\t# compressed ptr" %}
7193
7194 ins_encode(aarch64_enc_strw(src, mem));
7195
7196 ins_pipe(istore_reg_mem);
7197 %}
7198
7199 instruct storeImmN0(immN0 zero, memory4 mem)
7200 %{
7201 match(Set mem (StoreN mem zero));
7202 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7203
7204 ins_cost(INSN_COST);
7205 format %{ "strw zr, $mem\t# compressed ptr" %}
7206
7207 ins_encode(aarch64_enc_strw0(mem));
7208
7209 ins_pipe(istore_mem);
7210 %}
7211
7212 // Store Float
7213 instruct storeF(vRegF src, memory4 mem)
7214 %{
7215 match(Set mem (StoreF mem src));
7216 predicate(!needs_releasing_store(n));
7217
7218 ins_cost(INSN_COST);
7219 format %{ "strs $src, $mem\t# float" %}
7220
7221 ins_encode( aarch64_enc_strs(src, mem) );
7222
7223 ins_pipe(pipe_class_memory);
7224 %}
7225
7226 // TODO
7227 // implement storeImmF0 and storeFImmPacked
7228
7229 // Store Double
7230 instruct storeD(vRegD src, memory8 mem)
7231 %{
7232 match(Set mem (StoreD mem src));
7233 predicate(!needs_releasing_store(n));
7234
7235 ins_cost(INSN_COST);
7236 format %{ "strd $src, $mem\t# double" %}
7237
7238 ins_encode( aarch64_enc_strd(src, mem) );
7239
7240 ins_pipe(pipe_class_memory);
7241 %}
7242
7243 // Store Compressed Klass Pointer
7244 instruct storeNKlass(iRegN src, memory4 mem)
7245 %{
7246 predicate(!needs_releasing_store(n));
7247 match(Set mem (StoreNKlass mem src));
7248
7249 ins_cost(INSN_COST);
7250 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7251
7252 ins_encode(aarch64_enc_strw(src, mem));
7253
7254 ins_pipe(istore_reg_mem);
7255 %}
7256
7257 // TODO
7258 // implement storeImmD0 and storeDImmPacked
7259
7260 // prefetch instructions
7261 // Must be safe to execute with invalid address (cannot fault).
7262
7263 instruct prefetchalloc( memory8 mem ) %{
7264 match(PrefetchAllocation mem);
7265
7266 ins_cost(INSN_COST);
7267 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7268
7269 ins_encode( aarch64_enc_prefetchw(mem) );
7270
7271 ins_pipe(iload_prefetch);
7272 %}
7273
7274 // ---------------- volatile loads and stores ----------------
7275
7276 // Load Byte (8 bit signed)
7277 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7278 %{
7279 match(Set dst (LoadB mem));
7280
7281 ins_cost(VOLATILE_REF_COST);
7282 format %{ "ldarsb $dst, $mem\t# byte" %}
7283
7284 ins_encode(aarch64_enc_ldarsb(dst, mem));
7285
7286 ins_pipe(pipe_serial);
7287 %}
7288
7289 // Load Byte (8 bit signed) into long
7290 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7291 %{
7292 match(Set dst (ConvI2L (LoadB mem)));
7293
7294 ins_cost(VOLATILE_REF_COST);
7295 format %{ "ldarsb $dst, $mem\t# byte" %}
7296
7297 ins_encode(aarch64_enc_ldarsb(dst, mem));
7298
7299 ins_pipe(pipe_serial);
7300 %}
7301
7302 // Load Byte (8 bit unsigned)
7303 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7304 %{
7305 match(Set dst (LoadUB mem));
7306
7307 ins_cost(VOLATILE_REF_COST);
7308 format %{ "ldarb $dst, $mem\t# byte" %}
7309
7310 ins_encode(aarch64_enc_ldarb(dst, mem));
7311
7312 ins_pipe(pipe_serial);
7313 %}
7314
7315 // Load Byte (8 bit unsigned) into long
7316 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7317 %{
7318 match(Set dst (ConvI2L (LoadUB mem)));
7319
7320 ins_cost(VOLATILE_REF_COST);
7321 format %{ "ldarb $dst, $mem\t# byte" %}
7322
7323 ins_encode(aarch64_enc_ldarb(dst, mem));
7324
7325 ins_pipe(pipe_serial);
7326 %}
7327
7328 // Load Short (16 bit signed)
7329 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7330 %{
7331 match(Set dst (LoadS mem));
7332
7333 ins_cost(VOLATILE_REF_COST);
7334 format %{ "ldarshw $dst, $mem\t# short" %}
7335
7336 ins_encode(aarch64_enc_ldarshw(dst, mem));
7337
7338 ins_pipe(pipe_serial);
7339 %}
7340
7341 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7342 %{
7343 match(Set dst (LoadUS mem));
7344
7345 ins_cost(VOLATILE_REF_COST);
7346 format %{ "ldarhw $dst, $mem\t# short" %}
7347
7348 ins_encode(aarch64_enc_ldarhw(dst, mem));
7349
7350 ins_pipe(pipe_serial);
7351 %}
7352
7353 // Load Short/Char (16 bit unsigned) into long
7354 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7355 %{
7356 match(Set dst (ConvI2L (LoadUS mem)));
7357
7358 ins_cost(VOLATILE_REF_COST);
7359 format %{ "ldarh $dst, $mem\t# short" %}
7360
7361 ins_encode(aarch64_enc_ldarh(dst, mem));
7362
7363 ins_pipe(pipe_serial);
7364 %}
7365
7366 // Load Short/Char (16 bit signed) into long
7367 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7368 %{
7369 match(Set dst (ConvI2L (LoadS mem)));
7370
7371 ins_cost(VOLATILE_REF_COST);
7372 format %{ "ldarh $dst, $mem\t# short" %}
7373
7374 ins_encode(aarch64_enc_ldarsh(dst, mem));
7375
7376 ins_pipe(pipe_serial);
7377 %}
7378
7379 // Load Integer (32 bit signed)
7380 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7381 %{
7382 match(Set dst (LoadI mem));
7383
7384 ins_cost(VOLATILE_REF_COST);
7385 format %{ "ldarw $dst, $mem\t# int" %}
7386
7387 ins_encode(aarch64_enc_ldarw(dst, mem));
7388
7389 ins_pipe(pipe_serial);
7390 %}
7391
7392 // Load Integer (32 bit unsigned) into long
7393 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7394 %{
7395 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7396
7397 ins_cost(VOLATILE_REF_COST);
7398 format %{ "ldarw $dst, $mem\t# int" %}
7399
7400 ins_encode(aarch64_enc_ldarw(dst, mem));
7401
7402 ins_pipe(pipe_serial);
7403 %}
7404
7405 // Load Long (64 bit signed)
7406 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7407 %{
7408 match(Set dst (LoadL mem));
7409
7410 ins_cost(VOLATILE_REF_COST);
7411 format %{ "ldar $dst, $mem\t# int" %}
7412
7413 ins_encode(aarch64_enc_ldar(dst, mem));
7414
7415 ins_pipe(pipe_serial);
7416 %}
7417
7418 // Load Pointer
7419 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7420 %{
7421 match(Set dst (LoadP mem));
7422 predicate(n->as_Load()->barrier_data() == 0);
7423
7424 ins_cost(VOLATILE_REF_COST);
7425 format %{ "ldar $dst, $mem\t# ptr" %}
7426
7427 ins_encode(aarch64_enc_ldar(dst, mem));
7428
7429 ins_pipe(pipe_serial);
7430 %}
7431
7432 // Load Compressed Pointer
7433 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7434 %{
7435 match(Set dst (LoadN mem));
7436 predicate(n->as_Load()->barrier_data() == 0);
7437
7438 ins_cost(VOLATILE_REF_COST);
7439 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7440
7441 ins_encode(aarch64_enc_ldarw(dst, mem));
7442
7443 ins_pipe(pipe_serial);
7444 %}
7445
7446 // Load Float
7447 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7448 %{
7449 match(Set dst (LoadF mem));
7450
7451 ins_cost(VOLATILE_REF_COST);
7452 format %{ "ldars $dst, $mem\t# float" %}
7453
7454 ins_encode( aarch64_enc_fldars(dst, mem) );
7455
7456 ins_pipe(pipe_serial);
7457 %}
7458
7459 // Load Double
7460 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7461 %{
7462 match(Set dst (LoadD mem));
7463
7464 ins_cost(VOLATILE_REF_COST);
7465 format %{ "ldard $dst, $mem\t# double" %}
7466
7467 ins_encode( aarch64_enc_fldard(dst, mem) );
7468
7469 ins_pipe(pipe_serial);
7470 %}
7471
7472 // Store Byte
7473 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7474 %{
7475 match(Set mem (StoreB mem src));
7476
7477 ins_cost(VOLATILE_REF_COST);
7478 format %{ "stlrb $src, $mem\t# byte" %}
7479
7480 ins_encode(aarch64_enc_stlrb(src, mem));
7481
7482 ins_pipe(pipe_class_memory);
7483 %}
7484
7485 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7486 %{
7487 match(Set mem (StoreB mem zero));
7488
7489 ins_cost(VOLATILE_REF_COST);
7490 format %{ "stlrb zr, $mem\t# byte" %}
7491
7492 ins_encode(aarch64_enc_stlrb0(mem));
7493
7494 ins_pipe(pipe_class_memory);
7495 %}
7496
7497 // Store Char/Short
7498 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7499 %{
7500 match(Set mem (StoreC mem src));
7501
7502 ins_cost(VOLATILE_REF_COST);
7503 format %{ "stlrh $src, $mem\t# short" %}
7504
7505 ins_encode(aarch64_enc_stlrh(src, mem));
7506
7507 ins_pipe(pipe_class_memory);
7508 %}
7509
7510 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7511 %{
7512 match(Set mem (StoreC mem zero));
7513
7514 ins_cost(VOLATILE_REF_COST);
7515 format %{ "stlrh zr, $mem\t# short" %}
7516
7517 ins_encode(aarch64_enc_stlrh0(mem));
7518
7519 ins_pipe(pipe_class_memory);
7520 %}
7521
7522 // Store Integer
7523
7524 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7525 %{
7526 match(Set mem(StoreI mem src));
7527
7528 ins_cost(VOLATILE_REF_COST);
7529 format %{ "stlrw $src, $mem\t# int" %}
7530
7531 ins_encode(aarch64_enc_stlrw(src, mem));
7532
7533 ins_pipe(pipe_class_memory);
7534 %}
7535
7536 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7537 %{
7538 match(Set mem(StoreI mem zero));
7539
7540 ins_cost(VOLATILE_REF_COST);
7541 format %{ "stlrw zr, $mem\t# int" %}
7542
7543 ins_encode(aarch64_enc_stlrw0(mem));
7544
7545 ins_pipe(pipe_class_memory);
7546 %}
7547
7548 // Store Long (64 bit signed)
7549 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7550 %{
7551 match(Set mem (StoreL mem src));
7552
7553 ins_cost(VOLATILE_REF_COST);
7554 format %{ "stlr $src, $mem\t# int" %}
7555
7556 ins_encode(aarch64_enc_stlr(src, mem));
7557
7558 ins_pipe(pipe_class_memory);
7559 %}
7560
7561 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7562 %{
7563 match(Set mem (StoreL mem zero));
7564
7565 ins_cost(VOLATILE_REF_COST);
7566 format %{ "stlr zr, $mem\t# int" %}
7567
7568 ins_encode(aarch64_enc_stlr0(mem));
7569
7570 ins_pipe(pipe_class_memory);
7571 %}
7572
7573 // Store Pointer
7574 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7575 %{
7576 match(Set mem (StoreP mem src));
7577 predicate(n->as_Store()->barrier_data() == 0);
7578
7579 ins_cost(VOLATILE_REF_COST);
7580 format %{ "stlr $src, $mem\t# ptr" %}
7581
7582 ins_encode(aarch64_enc_stlr(src, mem));
7583
7584 ins_pipe(pipe_class_memory);
7585 %}
7586
7587 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7588 %{
7589 match(Set mem (StoreP mem zero));
7590 predicate(n->as_Store()->barrier_data() == 0);
7591
7592 ins_cost(VOLATILE_REF_COST);
7593 format %{ "stlr zr, $mem\t# ptr" %}
7594
7595 ins_encode(aarch64_enc_stlr0(mem));
7596
7597 ins_pipe(pipe_class_memory);
7598 %}
7599
7600 // Store Compressed Pointer
7601 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7602 %{
7603 match(Set mem (StoreN mem src));
7604 predicate(n->as_Store()->barrier_data() == 0);
7605
7606 ins_cost(VOLATILE_REF_COST);
7607 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7608
7609 ins_encode(aarch64_enc_stlrw(src, mem));
7610
7611 ins_pipe(pipe_class_memory);
7612 %}
7613
7614 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7615 %{
7616 match(Set mem (StoreN mem zero));
7617 predicate(n->as_Store()->barrier_data() == 0);
7618
7619 ins_cost(VOLATILE_REF_COST);
7620 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7621
7622 ins_encode(aarch64_enc_stlrw0(mem));
7623
7624 ins_pipe(pipe_class_memory);
7625 %}
7626
7627 // Store Float
7628 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7629 %{
7630 match(Set mem (StoreF mem src));
7631
7632 ins_cost(VOLATILE_REF_COST);
7633 format %{ "stlrs $src, $mem\t# float" %}
7634
7635 ins_encode( aarch64_enc_fstlrs(src, mem) );
7636
7637 ins_pipe(pipe_class_memory);
7638 %}
7639
7640 // TODO
7641 // implement storeImmF0 and storeFImmPacked
7642
7643 // Store Double
7644 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7645 %{
7646 match(Set mem (StoreD mem src));
7647
7648 ins_cost(VOLATILE_REF_COST);
7649 format %{ "stlrd $src, $mem\t# double" %}
7650
7651 ins_encode( aarch64_enc_fstlrd(src, mem) );
7652
7653 ins_pipe(pipe_class_memory);
7654 %}
7655
7656 // ---------------- end of volatile loads and stores ----------------
7657
7658 instruct cacheWB(indirect addr)
7659 %{
7660 predicate(VM_Version::supports_data_cache_line_flush());
7661 match(CacheWB addr);
7662
7663 ins_cost(100);
7664 format %{"cache wb $addr" %}
7665 ins_encode %{
7666 assert($addr->index_position() < 0, "should be");
7667 assert($addr$$disp == 0, "should be");
7668 __ cache_wb(Address($addr$$base$$Register, 0));
7669 %}
7670 ins_pipe(pipe_slow); // XXX
7671 %}
7672
7673 instruct cacheWBPreSync()
7674 %{
7675 predicate(VM_Version::supports_data_cache_line_flush());
7676 match(CacheWBPreSync);
7677
7678 ins_cost(100);
7679 format %{"cache wb presync" %}
7680 ins_encode %{
7681 __ cache_wbsync(true);
7682 %}
7683 ins_pipe(pipe_slow); // XXX
7684 %}
7685
7686 instruct cacheWBPostSync()
7687 %{
7688 predicate(VM_Version::supports_data_cache_line_flush());
7689 match(CacheWBPostSync);
7690
7691 ins_cost(100);
7692 format %{"cache wb postsync" %}
7693 ins_encode %{
7694 __ cache_wbsync(false);
7695 %}
7696 ins_pipe(pipe_slow); // XXX
7697 %}
7698
7699 // ============================================================================
7700 // BSWAP Instructions
7701
7702 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7703 match(Set dst (ReverseBytesI src));
7704
7705 ins_cost(INSN_COST);
7706 format %{ "revw $dst, $src" %}
7707
7708 ins_encode %{
7709 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7710 %}
7711
7712 ins_pipe(ialu_reg);
7713 %}
7714
7715 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7716 match(Set dst (ReverseBytesL src));
7717
7718 ins_cost(INSN_COST);
7719 format %{ "rev $dst, $src" %}
7720
7721 ins_encode %{
7722 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7723 %}
7724
7725 ins_pipe(ialu_reg);
7726 %}
7727
7728 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7729 match(Set dst (ReverseBytesUS src));
7730
7731 ins_cost(INSN_COST);
7732 format %{ "rev16w $dst, $src\t# $dst -> unsigned short" %}
7733
7734 ins_encode %{
7735 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7736 __ narrow_subword_type(as_Register($dst$$reg), T_CHAR);
7737 %}
7738
7739 ins_pipe(ialu_reg);
7740 %}
7741
7742 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7743 match(Set dst (ReverseBytesS src));
7744
7745 ins_cost(INSN_COST);
7746 format %{ "rev16w $dst, $src\n\t"
7747 "sbfmw $dst, $dst, #0, #15" %}
7748
7749 ins_encode %{
7750 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7751 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7752 %}
7753
7754 ins_pipe(ialu_reg);
7755 %}
7756
7757 // ============================================================================
7758 // Zero Count Instructions
7759
7760 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7761 match(Set dst (CountLeadingZerosI src));
7762
7763 ins_cost(INSN_COST);
7764 format %{ "clzw $dst, $src" %}
7765 ins_encode %{
7766 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7767 %}
7768
7769 ins_pipe(ialu_reg);
7770 %}
7771
7772 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7773 match(Set dst (CountLeadingZerosL src));
7774
7775 ins_cost(INSN_COST);
7776 format %{ "clz $dst, $src" %}
7777 ins_encode %{
7778 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7779 %}
7780
7781 ins_pipe(ialu_reg);
7782 %}
7783
7784 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7785 match(Set dst (CountTrailingZerosI src));
7786
7787 ins_cost(INSN_COST * 2);
7788 format %{ "rbitw $dst, $src\n\t"
7789 "clzw $dst, $dst" %}
7790 ins_encode %{
7791 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7792 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7793 %}
7794
7795 ins_pipe(ialu_reg);
7796 %}
7797
7798 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7799 match(Set dst (CountTrailingZerosL src));
7800
7801 ins_cost(INSN_COST * 2);
7802 format %{ "rbit $dst, $src\n\t"
7803 "clz $dst, $dst" %}
7804 ins_encode %{
7805 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7806 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7807 %}
7808
7809 ins_pipe(ialu_reg);
7810 %}
7811
7812 //---------- Population Count Instructions -------------------------------------
7813 //
7814
7815 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7816 match(Set dst (PopCountI src));
7817 effect(TEMP tmp);
7818 ins_cost(INSN_COST * 13);
7819
7820 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7821 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7822 "addv $tmp, $tmp\t# vector (8B)\n\t"
7823 "mov $dst, $tmp\t# vector (1D)" %}
7824 ins_encode %{
7825 __ fmovs($tmp$$FloatRegister, $src$$Register);
7826 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7827 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7828 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7829 %}
7830
7831 ins_pipe(pipe_class_default);
7832 %}
7833
7834 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7835 match(Set dst (PopCountI (LoadI mem)));
7836 effect(TEMP tmp);
7837 ins_cost(INSN_COST * 13);
7838
7839 format %{ "ldrs $tmp, $mem\n\t"
7840 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7841 "addv $tmp, $tmp\t# vector (8B)\n\t"
7842 "mov $dst, $tmp\t# vector (1D)" %}
7843 ins_encode %{
7844 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7845 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7846 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7847 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7848 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7849 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7850 %}
7851
7852 ins_pipe(pipe_class_default);
7853 %}
7854
7855 // Note: Long.bitCount(long) returns an int.
7856 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7857 match(Set dst (PopCountL src));
7858 effect(TEMP tmp);
7859 ins_cost(INSN_COST * 13);
7860
7861 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7862 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7863 "addv $tmp, $tmp\t# vector (8B)\n\t"
7864 "mov $dst, $tmp\t# vector (1D)" %}
7865 ins_encode %{
7866 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7867 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7868 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7869 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7870 %}
7871
7872 ins_pipe(pipe_class_default);
7873 %}
7874
7875 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7876 match(Set dst (PopCountL (LoadL mem)));
7877 effect(TEMP tmp);
7878 ins_cost(INSN_COST * 13);
7879
7880 format %{ "ldrd $tmp, $mem\n\t"
7881 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7882 "addv $tmp, $tmp\t# vector (8B)\n\t"
7883 "mov $dst, $tmp\t# vector (1D)" %}
7884 ins_encode %{
7885 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7886 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7887 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7888 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7889 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7890 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7891 %}
7892
7893 ins_pipe(pipe_class_default);
7894 %}
7895
7896 // ============================================================================
7897 // VerifyVectorAlignment Instruction
7898
7899 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7900 match(Set addr (VerifyVectorAlignment addr mask));
7901 effect(KILL cr);
7902 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7903 ins_encode %{
7904 Label Lskip;
7905 // check if masked bits of addr are zero
7906 __ tst($addr$$Register, $mask$$constant);
7907 __ br(Assembler::EQ, Lskip);
7908 __ stop("verify_vector_alignment found a misaligned vector memory access");
7909 __ bind(Lskip);
7910 %}
7911 ins_pipe(pipe_slow);
7912 %}
7913
7914 // ============================================================================
7915 // MemBar Instruction
7916
7917 instruct load_fence() %{
7918 match(LoadFence);
7919 ins_cost(VOLATILE_REF_COST);
7920
7921 format %{ "load_fence" %}
7922
7923 ins_encode %{
7924 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7925 %}
7926 ins_pipe(pipe_serial);
7927 %}
7928
7929 instruct unnecessary_membar_acquire() %{
7930 predicate(unnecessary_acquire(n));
7931 match(MemBarAcquire);
7932 ins_cost(0);
7933
7934 format %{ "membar_acquire (elided)" %}
7935
7936 ins_encode %{
7937 __ block_comment("membar_acquire (elided)");
7938 %}
7939
7940 ins_pipe(pipe_class_empty);
7941 %}
7942
7943 instruct membar_acquire() %{
7944 match(MemBarAcquire);
7945 ins_cost(VOLATILE_REF_COST);
7946
7947 format %{ "membar_acquire\n\t"
7948 "dmb ishld" %}
7949
7950 ins_encode %{
7951 __ block_comment("membar_acquire");
7952 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7953 %}
7954
7955 ins_pipe(pipe_serial);
7956 %}
7957
7958
7959 instruct membar_acquire_lock() %{
7960 match(MemBarAcquireLock);
7961 ins_cost(VOLATILE_REF_COST);
7962
7963 format %{ "membar_acquire_lock (elided)" %}
7964
7965 ins_encode %{
7966 __ block_comment("membar_acquire_lock (elided)");
7967 %}
7968
7969 ins_pipe(pipe_serial);
7970 %}
7971
7972 instruct store_fence() %{
7973 match(StoreFence);
7974 ins_cost(VOLATILE_REF_COST);
7975
7976 format %{ "store_fence" %}
7977
7978 ins_encode %{
7979 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7980 %}
7981 ins_pipe(pipe_serial);
7982 %}
7983
7984 instruct unnecessary_membar_release() %{
7985 predicate(unnecessary_release(n));
7986 match(MemBarRelease);
7987 ins_cost(0);
7988
7989 format %{ "membar_release (elided)" %}
7990
7991 ins_encode %{
7992 __ block_comment("membar_release (elided)");
7993 %}
7994 ins_pipe(pipe_serial);
7995 %}
7996
7997 instruct membar_release() %{
7998 match(MemBarRelease);
7999 ins_cost(VOLATILE_REF_COST);
8000
8001 format %{ "membar_release\n\t"
8002 "dmb ishst\n\tdmb ishld" %}
8003
8004 ins_encode %{
8005 __ block_comment("membar_release");
8006 // These will be merged if AlwaysMergeDMB is enabled.
8007 __ membar(Assembler::StoreStore);
8008 __ membar(Assembler::LoadStore);
8009 %}
8010 ins_pipe(pipe_serial);
8011 %}
8012
8013 instruct membar_storestore() %{
8014 match(MemBarStoreStore);
8015 match(StoreStoreFence);
8016 ins_cost(VOLATILE_REF_COST);
8017
8018 format %{ "MEMBAR-store-store" %}
8019
8020 ins_encode %{
8021 __ membar(Assembler::StoreStore);
8022 %}
8023 ins_pipe(pipe_serial);
8024 %}
8025
8026 instruct membar_release_lock() %{
8027 match(MemBarReleaseLock);
8028 ins_cost(VOLATILE_REF_COST);
8029
8030 format %{ "membar_release_lock (elided)" %}
8031
8032 ins_encode %{
8033 __ block_comment("membar_release_lock (elided)");
8034 %}
8035
8036 ins_pipe(pipe_serial);
8037 %}
8038
8039 instruct membar_storeload() %{
8040 match(MemBarStoreLoad);
8041 ins_cost(VOLATILE_REF_COST*100);
8042
8043 format %{ "MEMBAR-store-load\n\t"
8044 "dmb ish" %}
8045
8046 ins_encode %{
8047 __ block_comment("membar_storeload");
8048 __ membar(Assembler::StoreLoad);
8049 %}
8050
8051 ins_pipe(pipe_serial);
8052 %}
8053
8054 instruct unnecessary_membar_volatile() %{
8055 predicate(unnecessary_volatile(n));
8056 match(MemBarVolatile);
8057 ins_cost(0);
8058
8059 format %{ "membar_volatile (elided)" %}
8060
8061 ins_encode %{
8062 __ block_comment("membar_volatile (elided)");
8063 %}
8064
8065 ins_pipe(pipe_serial);
8066 %}
8067
8068 instruct membar_volatile() %{
8069 match(MemBarVolatile);
8070 ins_cost(VOLATILE_REF_COST*100);
8071
8072 format %{ "membar_volatile\n\t"
8073 "dmb ish"%}
8074
8075 ins_encode %{
8076 __ block_comment("membar_volatile");
8077 __ membar(Assembler::StoreLoad);
8078 %}
8079
8080 ins_pipe(pipe_serial);
8081 %}
8082
8083 instruct membar_full() %{
8084 match(MemBarFull);
8085 ins_cost(VOLATILE_REF_COST*100);
8086
8087 format %{ "membar_full\n\t"
8088 "dmb ish" %}
8089 ins_encode %{
8090 __ block_comment("membar_full");
8091 __ membar(Assembler::AnyAny);
8092 %}
8093
8094 ins_pipe(pipe_serial);
8095 %}
8096
8097 // ============================================================================
8098 // Cast/Convert Instructions
8099
8100 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8101 match(Set dst (CastX2P src));
8102
8103 ins_cost(INSN_COST);
8104 format %{ "mov $dst, $src\t# long -> ptr" %}
8105
8106 ins_encode %{
8107 if ($dst$$reg != $src$$reg) {
8108 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8109 }
8110 %}
8111
8112 ins_pipe(ialu_reg);
8113 %}
8114
8115 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8116 match(Set dst (CastP2X src));
8117
8118 ins_cost(INSN_COST);
8119 format %{ "mov $dst, $src\t# ptr -> long" %}
8120
8121 ins_encode %{
8122 if ($dst$$reg != $src$$reg) {
8123 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8124 }
8125 %}
8126
8127 ins_pipe(ialu_reg);
8128 %}
8129
8130 // Convert oop into int for vectors alignment masking
8131 instruct convP2I(iRegINoSp dst, iRegP src) %{
8132 match(Set dst (ConvL2I (CastP2X src)));
8133
8134 ins_cost(INSN_COST);
8135 format %{ "movw $dst, $src\t# ptr -> int" %}
8136 ins_encode %{
8137 __ movw($dst$$Register, $src$$Register);
8138 %}
8139
8140 ins_pipe(ialu_reg);
8141 %}
8142
8143 // Convert compressed oop into int for vectors alignment masking
8144 // in case of 32bit oops (heap < 4Gb).
8145 instruct convN2I(iRegINoSp dst, iRegN src)
8146 %{
8147 predicate(CompressedOops::shift() == 0);
8148 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8149
8150 ins_cost(INSN_COST);
8151 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8152 ins_encode %{
8153 __ movw($dst$$Register, $src$$Register);
8154 %}
8155
8156 ins_pipe(ialu_reg);
8157 %}
8158
8159
8160 // Convert oop pointer into compressed form
8161 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8162 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8163 match(Set dst (EncodeP src));
8164 effect(KILL cr);
8165 ins_cost(INSN_COST * 3);
8166 format %{ "encode_heap_oop $dst, $src" %}
8167 ins_encode %{
8168 Register s = $src$$Register;
8169 Register d = $dst$$Register;
8170 __ encode_heap_oop(d, s);
8171 %}
8172 ins_pipe(ialu_reg);
8173 %}
8174
8175 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8176 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8177 match(Set dst (EncodeP src));
8178 ins_cost(INSN_COST * 3);
8179 format %{ "encode_heap_oop_not_null $dst, $src" %}
8180 ins_encode %{
8181 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8182 %}
8183 ins_pipe(ialu_reg);
8184 %}
8185
8186 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8187 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8188 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8189 match(Set dst (DecodeN src));
8190 ins_cost(INSN_COST * 3);
8191 format %{ "decode_heap_oop $dst, $src" %}
8192 ins_encode %{
8193 Register s = $src$$Register;
8194 Register d = $dst$$Register;
8195 __ decode_heap_oop(d, s);
8196 %}
8197 ins_pipe(ialu_reg);
8198 %}
8199
8200 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8201 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8202 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8203 match(Set dst (DecodeN src));
8204 ins_cost(INSN_COST * 3);
8205 format %{ "decode_heap_oop_not_null $dst, $src" %}
8206 ins_encode %{
8207 Register s = $src$$Register;
8208 Register d = $dst$$Register;
8209 __ decode_heap_oop_not_null(d, s);
8210 %}
8211 ins_pipe(ialu_reg);
8212 %}
8213
8214 // n.b. AArch64 implementations of encode_klass_not_null and
8215 // decode_klass_not_null do not modify the flags register so, unlike
8216 // Intel, we don't kill CR as a side effect here
8217
8218 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8219 match(Set dst (EncodePKlass src));
8220
8221 ins_cost(INSN_COST * 3);
8222 format %{ "encode_klass_not_null $dst,$src" %}
8223
8224 ins_encode %{
8225 Register src_reg = as_Register($src$$reg);
8226 Register dst_reg = as_Register($dst$$reg);
8227 __ encode_klass_not_null(dst_reg, src_reg);
8228 %}
8229
8230 ins_pipe(ialu_reg);
8231 %}
8232
8233 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8234 match(Set dst (DecodeNKlass src));
8235
8236 ins_cost(INSN_COST * 3);
8237 format %{ "decode_klass_not_null $dst,$src" %}
8238
8239 ins_encode %{
8240 Register src_reg = as_Register($src$$reg);
8241 Register dst_reg = as_Register($dst$$reg);
8242 if (dst_reg != src_reg) {
8243 __ decode_klass_not_null(dst_reg, src_reg);
8244 } else {
8245 __ decode_klass_not_null(dst_reg);
8246 }
8247 %}
8248
8249 ins_pipe(ialu_reg);
8250 %}
8251
8252 instruct checkCastPP(iRegPNoSp dst)
8253 %{
8254 match(Set dst (CheckCastPP dst));
8255
8256 size(0);
8257 format %{ "# checkcastPP of $dst" %}
8258 ins_encode(/* empty encoding */);
8259 ins_pipe(pipe_class_empty);
8260 %}
8261
8262 instruct castPP(iRegPNoSp dst)
8263 %{
8264 match(Set dst (CastPP dst));
8265
8266 size(0);
8267 format %{ "# castPP of $dst" %}
8268 ins_encode(/* empty encoding */);
8269 ins_pipe(pipe_class_empty);
8270 %}
8271
8272 instruct castII(iRegI dst)
8273 %{
8274 predicate(VerifyConstraintCasts == 0);
8275 match(Set dst (CastII dst));
8276
8277 size(0);
8278 format %{ "# castII of $dst" %}
8279 ins_encode(/* empty encoding */);
8280 ins_cost(0);
8281 ins_pipe(pipe_class_empty);
8282 %}
8283
8284 instruct castII_checked(iRegI dst, rFlagsReg cr)
8285 %{
8286 predicate(VerifyConstraintCasts > 0);
8287 match(Set dst (CastII dst));
8288 effect(KILL cr);
8289
8290 format %{ "# castII_checked of $dst" %}
8291 ins_encode %{
8292 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8293 %}
8294 ins_pipe(pipe_slow);
8295 %}
8296
8297 // The unchecked and checked variants for CastII below both use iRegINoSp for src and dst
8298 // as some consumers of CastII node like ConvHF2F forbid the stack pointer as an input
8299 // (please see convHF2F_reg_reg rule which requires input to be in an iRegINoSp register).
8300 instruct castII_nosp(iRegINoSp dst)
8301 %{
8302 predicate(VerifyConstraintCasts == 0);
8303 match(Set dst (CastII dst));
8304
8305 size(0);
8306 format %{ "# castII of $dst" %}
8307 ins_encode(/* empty encoding */);
8308 ins_cost(0);
8309 ins_pipe(pipe_class_empty);
8310 %}
8311
8312 instruct castII_checked_nosp(iRegINoSp dst, rFlagsReg cr)
8313 %{
8314 predicate(VerifyConstraintCasts > 0);
8315 match(Set dst (CastII dst));
8316 effect(KILL cr);
8317
8318 format %{ "# castII_checked of $dst" %}
8319 ins_encode %{
8320 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8321 %}
8322 ins_pipe(pipe_slow);
8323 %}
8324
8325 instruct castLL(iRegL dst)
8326 %{
8327 predicate(VerifyConstraintCasts == 0);
8328 match(Set dst (CastLL dst));
8329
8330 size(0);
8331 format %{ "# castLL of $dst" %}
8332 ins_encode(/* empty encoding */);
8333 ins_cost(0);
8334 ins_pipe(pipe_class_empty);
8335 %}
8336
8337 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8338 %{
8339 predicate(VerifyConstraintCasts > 0);
8340 match(Set dst (CastLL dst));
8341 effect(KILL cr);
8342
8343 format %{ "# castLL_checked of $dst" %}
8344 ins_encode %{
8345 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8346 %}
8347 ins_pipe(pipe_slow);
8348 %}
8349
8350 instruct castHH(vRegF dst)
8351 %{
8352 match(Set dst (CastHH dst));
8353 size(0);
8354 format %{ "# castHH of $dst" %}
8355 ins_encode(/* empty encoding */);
8356 ins_cost(0);
8357 ins_pipe(pipe_class_empty);
8358 %}
8359
8360 instruct castFF(vRegF dst)
8361 %{
8362 match(Set dst (CastFF dst));
8363
8364 size(0);
8365 format %{ "# castFF of $dst" %}
8366 ins_encode(/* empty encoding */);
8367 ins_cost(0);
8368 ins_pipe(pipe_class_empty);
8369 %}
8370
8371 instruct castDD(vRegD dst)
8372 %{
8373 match(Set dst (CastDD dst));
8374
8375 size(0);
8376 format %{ "# castDD of $dst" %}
8377 ins_encode(/* empty encoding */);
8378 ins_cost(0);
8379 ins_pipe(pipe_class_empty);
8380 %}
8381
8382 instruct castVV(vReg dst)
8383 %{
8384 match(Set dst (CastVV dst));
8385
8386 size(0);
8387 format %{ "# castVV of $dst" %}
8388 ins_encode(/* empty encoding */);
8389 ins_cost(0);
8390 ins_pipe(pipe_class_empty);
8391 %}
8392
8393 instruct castVVMask(pRegGov dst)
8394 %{
8395 match(Set dst (CastVV dst));
8396
8397 size(0);
8398 format %{ "# castVV of $dst" %}
8399 ins_encode(/* empty encoding */);
8400 ins_cost(0);
8401 ins_pipe(pipe_class_empty);
8402 %}
8403
8404 // Manifest a CmpU result in an integer register.
8405 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8406 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8407 %{
8408 match(Set dst (CmpU3 src1 src2));
8409 effect(KILL flags);
8410
8411 ins_cost(INSN_COST * 3);
8412 format %{
8413 "cmpw $src1, $src2\n\t"
8414 "csetw $dst, ne\n\t"
8415 "cnegw $dst, lo\t# CmpU3(reg)"
8416 %}
8417 ins_encode %{
8418 __ cmpw($src1$$Register, $src2$$Register);
8419 __ csetw($dst$$Register, Assembler::NE);
8420 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8421 %}
8422
8423 ins_pipe(pipe_class_default);
8424 %}
8425
8426 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8427 %{
8428 match(Set dst (CmpU3 src1 src2));
8429 effect(KILL flags);
8430
8431 ins_cost(INSN_COST * 3);
8432 format %{
8433 "subsw zr, $src1, $src2\n\t"
8434 "csetw $dst, ne\n\t"
8435 "cnegw $dst, lo\t# CmpU3(imm)"
8436 %}
8437 ins_encode %{
8438 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8439 __ csetw($dst$$Register, Assembler::NE);
8440 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8441 %}
8442
8443 ins_pipe(pipe_class_default);
8444 %}
8445
8446 // Manifest a CmpUL result in an integer register.
8447 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8448 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8449 %{
8450 match(Set dst (CmpUL3 src1 src2));
8451 effect(KILL flags);
8452
8453 ins_cost(INSN_COST * 3);
8454 format %{
8455 "cmp $src1, $src2\n\t"
8456 "csetw $dst, ne\n\t"
8457 "cnegw $dst, lo\t# CmpUL3(reg)"
8458 %}
8459 ins_encode %{
8460 __ cmp($src1$$Register, $src2$$Register);
8461 __ csetw($dst$$Register, Assembler::NE);
8462 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8463 %}
8464
8465 ins_pipe(pipe_class_default);
8466 %}
8467
8468 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8469 %{
8470 match(Set dst (CmpUL3 src1 src2));
8471 effect(KILL flags);
8472
8473 ins_cost(INSN_COST * 3);
8474 format %{
8475 "subs zr, $src1, $src2\n\t"
8476 "csetw $dst, ne\n\t"
8477 "cnegw $dst, lo\t# CmpUL3(imm)"
8478 %}
8479 ins_encode %{
8480 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8481 __ csetw($dst$$Register, Assembler::NE);
8482 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8483 %}
8484
8485 ins_pipe(pipe_class_default);
8486 %}
8487
8488 // Manifest a CmpL result in an integer register.
8489 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8490 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8491 %{
8492 match(Set dst (CmpL3 src1 src2));
8493 effect(KILL flags);
8494
8495 ins_cost(INSN_COST * 3);
8496 format %{
8497 "cmp $src1, $src2\n\t"
8498 "csetw $dst, ne\n\t"
8499 "cnegw $dst, lt\t# CmpL3(reg)"
8500 %}
8501 ins_encode %{
8502 __ cmp($src1$$Register, $src2$$Register);
8503 __ csetw($dst$$Register, Assembler::NE);
8504 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8505 %}
8506
8507 ins_pipe(pipe_class_default);
8508 %}
8509
8510 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8511 %{
8512 match(Set dst (CmpL3 src1 src2));
8513 effect(KILL flags);
8514
8515 ins_cost(INSN_COST * 3);
8516 format %{
8517 "subs zr, $src1, $src2\n\t"
8518 "csetw $dst, ne\n\t"
8519 "cnegw $dst, lt\t# CmpL3(imm)"
8520 %}
8521 ins_encode %{
8522 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8523 __ csetw($dst$$Register, Assembler::NE);
8524 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8525 %}
8526
8527 ins_pipe(pipe_class_default);
8528 %}
8529
8530 // ============================================================================
8531 // Conditional Move Instructions
8532
8533 // n.b. we have identical rules for both a signed compare op (cmpOp)
8534 // and an unsigned compare op (cmpOpU). it would be nice if we could
8535 // define an op class which merged both inputs and use it to type the
8536 // argument to a single rule. unfortunatelyt his fails because the
8537 // opclass does not live up to the COND_INTER interface of its
8538 // component operands. When the generic code tries to negate the
8539 // operand it ends up running the generci Machoper::negate method
8540 // which throws a ShouldNotHappen. So, we have to provide two flavours
8541 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8542
8543 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8544 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8545
8546 ins_cost(INSN_COST * 2);
8547 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8548
8549 ins_encode %{
8550 __ cselw(as_Register($dst$$reg),
8551 as_Register($src2$$reg),
8552 as_Register($src1$$reg),
8553 (Assembler::Condition)$cmp$$cmpcode);
8554 %}
8555
8556 ins_pipe(icond_reg_reg);
8557 %}
8558
8559 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8560 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8561
8562 ins_cost(INSN_COST * 2);
8563 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8564
8565 ins_encode %{
8566 __ cselw(as_Register($dst$$reg),
8567 as_Register($src2$$reg),
8568 as_Register($src1$$reg),
8569 (Assembler::Condition)$cmp$$cmpcode);
8570 %}
8571
8572 ins_pipe(icond_reg_reg);
8573 %}
8574
8575 // special cases where one arg is zero
8576
8577 // n.b. this is selected in preference to the rule above because it
8578 // avoids loading constant 0 into a source register
8579
8580 // TODO
8581 // we ought only to be able to cull one of these variants as the ideal
8582 // transforms ought always to order the zero consistently (to left/right?)
8583
8584 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8585 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8586
8587 ins_cost(INSN_COST * 2);
8588 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8589
8590 ins_encode %{
8591 __ cselw(as_Register($dst$$reg),
8592 as_Register($src$$reg),
8593 zr,
8594 (Assembler::Condition)$cmp$$cmpcode);
8595 %}
8596
8597 ins_pipe(icond_reg);
8598 %}
8599
8600 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8601 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8602
8603 ins_cost(INSN_COST * 2);
8604 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8605
8606 ins_encode %{
8607 __ cselw(as_Register($dst$$reg),
8608 as_Register($src$$reg),
8609 zr,
8610 (Assembler::Condition)$cmp$$cmpcode);
8611 %}
8612
8613 ins_pipe(icond_reg);
8614 %}
8615
8616 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8617 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8618
8619 ins_cost(INSN_COST * 2);
8620 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8621
8622 ins_encode %{
8623 __ cselw(as_Register($dst$$reg),
8624 zr,
8625 as_Register($src$$reg),
8626 (Assembler::Condition)$cmp$$cmpcode);
8627 %}
8628
8629 ins_pipe(icond_reg);
8630 %}
8631
8632 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8633 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8634
8635 ins_cost(INSN_COST * 2);
8636 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8637
8638 ins_encode %{
8639 __ cselw(as_Register($dst$$reg),
8640 zr,
8641 as_Register($src$$reg),
8642 (Assembler::Condition)$cmp$$cmpcode);
8643 %}
8644
8645 ins_pipe(icond_reg);
8646 %}
8647
8648 // special case for creating a boolean 0 or 1
8649
8650 // n.b. this is selected in preference to the rule above because it
8651 // avoids loading constants 0 and 1 into a source register
8652
8653 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8654 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8655
8656 ins_cost(INSN_COST * 2);
8657 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8658
8659 ins_encode %{
8660 // equivalently
8661 // cset(as_Register($dst$$reg),
8662 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8663 __ csincw(as_Register($dst$$reg),
8664 zr,
8665 zr,
8666 (Assembler::Condition)$cmp$$cmpcode);
8667 %}
8668
8669 ins_pipe(icond_none);
8670 %}
8671
8672 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8673 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8674
8675 ins_cost(INSN_COST * 2);
8676 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8677
8678 ins_encode %{
8679 // equivalently
8680 // cset(as_Register($dst$$reg),
8681 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8682 __ csincw(as_Register($dst$$reg),
8683 zr,
8684 zr,
8685 (Assembler::Condition)$cmp$$cmpcode);
8686 %}
8687
8688 ins_pipe(icond_none);
8689 %}
8690
8691 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8692 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8693
8694 ins_cost(INSN_COST * 2);
8695 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8696
8697 ins_encode %{
8698 __ csel(as_Register($dst$$reg),
8699 as_Register($src2$$reg),
8700 as_Register($src1$$reg),
8701 (Assembler::Condition)$cmp$$cmpcode);
8702 %}
8703
8704 ins_pipe(icond_reg_reg);
8705 %}
8706
8707 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8708 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8709
8710 ins_cost(INSN_COST * 2);
8711 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8712
8713 ins_encode %{
8714 __ csel(as_Register($dst$$reg),
8715 as_Register($src2$$reg),
8716 as_Register($src1$$reg),
8717 (Assembler::Condition)$cmp$$cmpcode);
8718 %}
8719
8720 ins_pipe(icond_reg_reg);
8721 %}
8722
8723 // special cases where one arg is zero
8724
8725 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8726 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8727
8728 ins_cost(INSN_COST * 2);
8729 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8730
8731 ins_encode %{
8732 __ csel(as_Register($dst$$reg),
8733 zr,
8734 as_Register($src$$reg),
8735 (Assembler::Condition)$cmp$$cmpcode);
8736 %}
8737
8738 ins_pipe(icond_reg);
8739 %}
8740
8741 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8742 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8743
8744 ins_cost(INSN_COST * 2);
8745 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8746
8747 ins_encode %{
8748 __ csel(as_Register($dst$$reg),
8749 zr,
8750 as_Register($src$$reg),
8751 (Assembler::Condition)$cmp$$cmpcode);
8752 %}
8753
8754 ins_pipe(icond_reg);
8755 %}
8756
8757 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8758 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8759
8760 ins_cost(INSN_COST * 2);
8761 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8762
8763 ins_encode %{
8764 __ csel(as_Register($dst$$reg),
8765 as_Register($src$$reg),
8766 zr,
8767 (Assembler::Condition)$cmp$$cmpcode);
8768 %}
8769
8770 ins_pipe(icond_reg);
8771 %}
8772
8773 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8774 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8775
8776 ins_cost(INSN_COST * 2);
8777 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8778
8779 ins_encode %{
8780 __ csel(as_Register($dst$$reg),
8781 as_Register($src$$reg),
8782 zr,
8783 (Assembler::Condition)$cmp$$cmpcode);
8784 %}
8785
8786 ins_pipe(icond_reg);
8787 %}
8788
8789 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8790 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8791
8792 ins_cost(INSN_COST * 2);
8793 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8794
8795 ins_encode %{
8796 __ csel(as_Register($dst$$reg),
8797 as_Register($src2$$reg),
8798 as_Register($src1$$reg),
8799 (Assembler::Condition)$cmp$$cmpcode);
8800 %}
8801
8802 ins_pipe(icond_reg_reg);
8803 %}
8804
8805 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8806 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8807
8808 ins_cost(INSN_COST * 2);
8809 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8810
8811 ins_encode %{
8812 __ csel(as_Register($dst$$reg),
8813 as_Register($src2$$reg),
8814 as_Register($src1$$reg),
8815 (Assembler::Condition)$cmp$$cmpcode);
8816 %}
8817
8818 ins_pipe(icond_reg_reg);
8819 %}
8820
8821 // special cases where one arg is zero
8822
8823 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8824 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8825
8826 ins_cost(INSN_COST * 2);
8827 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8828
8829 ins_encode %{
8830 __ csel(as_Register($dst$$reg),
8831 zr,
8832 as_Register($src$$reg),
8833 (Assembler::Condition)$cmp$$cmpcode);
8834 %}
8835
8836 ins_pipe(icond_reg);
8837 %}
8838
8839 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8840 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8841
8842 ins_cost(INSN_COST * 2);
8843 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8844
8845 ins_encode %{
8846 __ csel(as_Register($dst$$reg),
8847 zr,
8848 as_Register($src$$reg),
8849 (Assembler::Condition)$cmp$$cmpcode);
8850 %}
8851
8852 ins_pipe(icond_reg);
8853 %}
8854
8855 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8856 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8857
8858 ins_cost(INSN_COST * 2);
8859 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8860
8861 ins_encode %{
8862 __ csel(as_Register($dst$$reg),
8863 as_Register($src$$reg),
8864 zr,
8865 (Assembler::Condition)$cmp$$cmpcode);
8866 %}
8867
8868 ins_pipe(icond_reg);
8869 %}
8870
8871 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8872 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8873
8874 ins_cost(INSN_COST * 2);
8875 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8876
8877 ins_encode %{
8878 __ csel(as_Register($dst$$reg),
8879 as_Register($src$$reg),
8880 zr,
8881 (Assembler::Condition)$cmp$$cmpcode);
8882 %}
8883
8884 ins_pipe(icond_reg);
8885 %}
8886
8887 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8888 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8889
8890 ins_cost(INSN_COST * 2);
8891 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8892
8893 ins_encode %{
8894 __ cselw(as_Register($dst$$reg),
8895 as_Register($src2$$reg),
8896 as_Register($src1$$reg),
8897 (Assembler::Condition)$cmp$$cmpcode);
8898 %}
8899
8900 ins_pipe(icond_reg_reg);
8901 %}
8902
8903 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8904 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8905
8906 ins_cost(INSN_COST * 2);
8907 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8908
8909 ins_encode %{
8910 __ cselw(as_Register($dst$$reg),
8911 as_Register($src2$$reg),
8912 as_Register($src1$$reg),
8913 (Assembler::Condition)$cmp$$cmpcode);
8914 %}
8915
8916 ins_pipe(icond_reg_reg);
8917 %}
8918
8919 // special cases where one arg is zero
8920
8921 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8922 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8923
8924 ins_cost(INSN_COST * 2);
8925 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8926
8927 ins_encode %{
8928 __ cselw(as_Register($dst$$reg),
8929 zr,
8930 as_Register($src$$reg),
8931 (Assembler::Condition)$cmp$$cmpcode);
8932 %}
8933
8934 ins_pipe(icond_reg);
8935 %}
8936
8937 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8938 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8939
8940 ins_cost(INSN_COST * 2);
8941 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8942
8943 ins_encode %{
8944 __ cselw(as_Register($dst$$reg),
8945 zr,
8946 as_Register($src$$reg),
8947 (Assembler::Condition)$cmp$$cmpcode);
8948 %}
8949
8950 ins_pipe(icond_reg);
8951 %}
8952
8953 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8954 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8955
8956 ins_cost(INSN_COST * 2);
8957 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8958
8959 ins_encode %{
8960 __ cselw(as_Register($dst$$reg),
8961 as_Register($src$$reg),
8962 zr,
8963 (Assembler::Condition)$cmp$$cmpcode);
8964 %}
8965
8966 ins_pipe(icond_reg);
8967 %}
8968
8969 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8970 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8971
8972 ins_cost(INSN_COST * 2);
8973 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8974
8975 ins_encode %{
8976 __ cselw(as_Register($dst$$reg),
8977 as_Register($src$$reg),
8978 zr,
8979 (Assembler::Condition)$cmp$$cmpcode);
8980 %}
8981
8982 ins_pipe(icond_reg);
8983 %}
8984
8985 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8986 %{
8987 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8988
8989 ins_cost(INSN_COST * 3);
8990
8991 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8992 ins_encode %{
8993 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8994 __ fcsels(as_FloatRegister($dst$$reg),
8995 as_FloatRegister($src2$$reg),
8996 as_FloatRegister($src1$$reg),
8997 cond);
8998 %}
8999
9000 ins_pipe(fp_cond_reg_reg_s);
9001 %}
9002
9003 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9004 %{
9005 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9006
9007 ins_cost(INSN_COST * 3);
9008
9009 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9010 ins_encode %{
9011 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9012 __ fcsels(as_FloatRegister($dst$$reg),
9013 as_FloatRegister($src2$$reg),
9014 as_FloatRegister($src1$$reg),
9015 cond);
9016 %}
9017
9018 ins_pipe(fp_cond_reg_reg_s);
9019 %}
9020
9021 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9022 %{
9023 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9024
9025 ins_cost(INSN_COST * 3);
9026
9027 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9028 ins_encode %{
9029 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9030 __ fcseld(as_FloatRegister($dst$$reg),
9031 as_FloatRegister($src2$$reg),
9032 as_FloatRegister($src1$$reg),
9033 cond);
9034 %}
9035
9036 ins_pipe(fp_cond_reg_reg_d);
9037 %}
9038
9039 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9040 %{
9041 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9042
9043 ins_cost(INSN_COST * 3);
9044
9045 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9046 ins_encode %{
9047 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9048 __ fcseld(as_FloatRegister($dst$$reg),
9049 as_FloatRegister($src2$$reg),
9050 as_FloatRegister($src1$$reg),
9051 cond);
9052 %}
9053
9054 ins_pipe(fp_cond_reg_reg_d);
9055 %}
9056
9057 // ============================================================================
9058 // Arithmetic Instructions
9059 //
9060
9061 // Integer Addition
9062
9063 // TODO
9064 // these currently employ operations which do not set CR and hence are
9065 // not flagged as killing CR but we would like to isolate the cases
9066 // where we want to set flags from those where we don't. need to work
9067 // out how to do that.
9068
9069 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9070 match(Set dst (AddI src1 src2));
9071
9072 ins_cost(INSN_COST);
9073 format %{ "addw $dst, $src1, $src2" %}
9074
9075 ins_encode %{
9076 __ addw(as_Register($dst$$reg),
9077 as_Register($src1$$reg),
9078 as_Register($src2$$reg));
9079 %}
9080
9081 ins_pipe(ialu_reg_reg);
9082 %}
9083
9084 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9085 match(Set dst (AddI src1 src2));
9086
9087 ins_cost(INSN_COST);
9088 format %{ "addw $dst, $src1, $src2" %}
9089
9090 // use opcode to indicate that this is an add not a sub
9091 opcode(0x0);
9092
9093 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9094
9095 ins_pipe(ialu_reg_imm);
9096 %}
9097
9098 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9099 match(Set dst (AddI (ConvL2I src1) src2));
9100
9101 ins_cost(INSN_COST);
9102 format %{ "addw $dst, $src1, $src2" %}
9103
9104 // use opcode to indicate that this is an add not a sub
9105 opcode(0x0);
9106
9107 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9108
9109 ins_pipe(ialu_reg_imm);
9110 %}
9111
9112 // Pointer Addition
9113 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9114 match(Set dst (AddP src1 src2));
9115
9116 ins_cost(INSN_COST);
9117 format %{ "add $dst, $src1, $src2\t# ptr" %}
9118
9119 ins_encode %{
9120 __ add(as_Register($dst$$reg),
9121 as_Register($src1$$reg),
9122 as_Register($src2$$reg));
9123 %}
9124
9125 ins_pipe(ialu_reg_reg);
9126 %}
9127
9128 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9129 match(Set dst (AddP src1 (ConvI2L src2)));
9130
9131 ins_cost(1.9 * INSN_COST);
9132 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9133
9134 ins_encode %{
9135 __ add(as_Register($dst$$reg),
9136 as_Register($src1$$reg),
9137 as_Register($src2$$reg), ext::sxtw);
9138 %}
9139
9140 ins_pipe(ialu_reg_reg);
9141 %}
9142
9143 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9144 match(Set dst (AddP src1 (LShiftL src2 scale)));
9145
9146 ins_cost(1.9 * INSN_COST);
9147 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9148
9149 ins_encode %{
9150 __ lea(as_Register($dst$$reg),
9151 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9152 Address::lsl($scale$$constant)));
9153 %}
9154
9155 ins_pipe(ialu_reg_reg_shift);
9156 %}
9157
9158 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9159 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9160
9161 ins_cost(1.9 * INSN_COST);
9162 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9163
9164 ins_encode %{
9165 __ lea(as_Register($dst$$reg),
9166 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9167 Address::sxtw($scale$$constant)));
9168 %}
9169
9170 ins_pipe(ialu_reg_reg_shift);
9171 %}
9172
9173 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9174 match(Set dst (LShiftL (ConvI2L src) scale));
9175
9176 ins_cost(INSN_COST);
9177 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9178
9179 ins_encode %{
9180 __ sbfiz(as_Register($dst$$reg),
9181 as_Register($src$$reg),
9182 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9183 %}
9184
9185 ins_pipe(ialu_reg_shift);
9186 %}
9187
9188 // Pointer Immediate Addition
9189 // n.b. this needs to be more expensive than using an indirect memory
9190 // operand
9191 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9192 match(Set dst (AddP src1 src2));
9193
9194 ins_cost(INSN_COST);
9195 format %{ "add $dst, $src1, $src2\t# ptr" %}
9196
9197 // use opcode to indicate that this is an add not a sub
9198 opcode(0x0);
9199
9200 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9201
9202 ins_pipe(ialu_reg_imm);
9203 %}
9204
9205 // Long Addition
9206 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9207
9208 match(Set dst (AddL src1 src2));
9209
9210 ins_cost(INSN_COST);
9211 format %{ "add $dst, $src1, $src2" %}
9212
9213 ins_encode %{
9214 __ add(as_Register($dst$$reg),
9215 as_Register($src1$$reg),
9216 as_Register($src2$$reg));
9217 %}
9218
9219 ins_pipe(ialu_reg_reg);
9220 %}
9221
9222 // No constant pool entries requiredLong Immediate Addition.
9223 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9224 match(Set dst (AddL src1 src2));
9225
9226 ins_cost(INSN_COST);
9227 format %{ "add $dst, $src1, $src2" %}
9228
9229 // use opcode to indicate that this is an add not a sub
9230 opcode(0x0);
9231
9232 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9233
9234 ins_pipe(ialu_reg_imm);
9235 %}
9236
9237 // Integer Subtraction
9238 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9239 match(Set dst (SubI src1 src2));
9240
9241 ins_cost(INSN_COST);
9242 format %{ "subw $dst, $src1, $src2" %}
9243
9244 ins_encode %{
9245 __ subw(as_Register($dst$$reg),
9246 as_Register($src1$$reg),
9247 as_Register($src2$$reg));
9248 %}
9249
9250 ins_pipe(ialu_reg_reg);
9251 %}
9252
9253 // Immediate Subtraction
9254 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9255 match(Set dst (SubI src1 src2));
9256
9257 ins_cost(INSN_COST);
9258 format %{ "subw $dst, $src1, $src2" %}
9259
9260 // use opcode to indicate that this is a sub not an add
9261 opcode(0x1);
9262
9263 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9264
9265 ins_pipe(ialu_reg_imm);
9266 %}
9267
9268 // Long Subtraction
9269 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9270
9271 match(Set dst (SubL src1 src2));
9272
9273 ins_cost(INSN_COST);
9274 format %{ "sub $dst, $src1, $src2" %}
9275
9276 ins_encode %{
9277 __ sub(as_Register($dst$$reg),
9278 as_Register($src1$$reg),
9279 as_Register($src2$$reg));
9280 %}
9281
9282 ins_pipe(ialu_reg_reg);
9283 %}
9284
9285 // No constant pool entries requiredLong Immediate Subtraction.
9286 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9287 match(Set dst (SubL src1 src2));
9288
9289 ins_cost(INSN_COST);
9290 format %{ "sub$dst, $src1, $src2" %}
9291
9292 // use opcode to indicate that this is a sub not an add
9293 opcode(0x1);
9294
9295 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9296
9297 ins_pipe(ialu_reg_imm);
9298 %}
9299
9300 // Integer Negation (special case for sub)
9301
9302 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9303 match(Set dst (SubI zero src));
9304
9305 ins_cost(INSN_COST);
9306 format %{ "negw $dst, $src\t# int" %}
9307
9308 ins_encode %{
9309 __ negw(as_Register($dst$$reg),
9310 as_Register($src$$reg));
9311 %}
9312
9313 ins_pipe(ialu_reg);
9314 %}
9315
9316 // Long Negation
9317
9318 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9319 match(Set dst (SubL zero src));
9320
9321 ins_cost(INSN_COST);
9322 format %{ "neg $dst, $src\t# long" %}
9323
9324 ins_encode %{
9325 __ neg(as_Register($dst$$reg),
9326 as_Register($src$$reg));
9327 %}
9328
9329 ins_pipe(ialu_reg);
9330 %}
9331
9332 // Integer Multiply
9333
9334 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9335 match(Set dst (MulI src1 src2));
9336
9337 ins_cost(INSN_COST * 3);
9338 format %{ "mulw $dst, $src1, $src2" %}
9339
9340 ins_encode %{
9341 __ mulw(as_Register($dst$$reg),
9342 as_Register($src1$$reg),
9343 as_Register($src2$$reg));
9344 %}
9345
9346 ins_pipe(imul_reg_reg);
9347 %}
9348
9349 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9350 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9351
9352 ins_cost(INSN_COST * 3);
9353 format %{ "smull $dst, $src1, $src2" %}
9354
9355 ins_encode %{
9356 __ smull(as_Register($dst$$reg),
9357 as_Register($src1$$reg),
9358 as_Register($src2$$reg));
9359 %}
9360
9361 ins_pipe(imul_reg_reg);
9362 %}
9363
9364 // Long Multiply
9365
9366 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9367 match(Set dst (MulL src1 src2));
9368
9369 ins_cost(INSN_COST * 5);
9370 format %{ "mul $dst, $src1, $src2" %}
9371
9372 ins_encode %{
9373 __ mul(as_Register($dst$$reg),
9374 as_Register($src1$$reg),
9375 as_Register($src2$$reg));
9376 %}
9377
9378 ins_pipe(lmul_reg_reg);
9379 %}
9380
9381 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9382 %{
9383 match(Set dst (MulHiL src1 src2));
9384
9385 ins_cost(INSN_COST * 7);
9386 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9387
9388 ins_encode %{
9389 __ smulh(as_Register($dst$$reg),
9390 as_Register($src1$$reg),
9391 as_Register($src2$$reg));
9392 %}
9393
9394 ins_pipe(lmul_reg_reg);
9395 %}
9396
9397 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9398 %{
9399 match(Set dst (UMulHiL src1 src2));
9400
9401 ins_cost(INSN_COST * 7);
9402 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9403
9404 ins_encode %{
9405 __ umulh(as_Register($dst$$reg),
9406 as_Register($src1$$reg),
9407 as_Register($src2$$reg));
9408 %}
9409
9410 ins_pipe(lmul_reg_reg);
9411 %}
9412
9413 // Combined Integer Multiply & Add/Sub
9414
9415 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9416 match(Set dst (AddI src3 (MulI src1 src2)));
9417
9418 ins_cost(INSN_COST * 3);
9419 format %{ "madd $dst, $src1, $src2, $src3" %}
9420
9421 ins_encode %{
9422 __ maddw(as_Register($dst$$reg),
9423 as_Register($src1$$reg),
9424 as_Register($src2$$reg),
9425 as_Register($src3$$reg));
9426 %}
9427
9428 ins_pipe(imac_reg_reg);
9429 %}
9430
9431 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9432 match(Set dst (SubI src3 (MulI src1 src2)));
9433
9434 ins_cost(INSN_COST * 3);
9435 format %{ "msub $dst, $src1, $src2, $src3" %}
9436
9437 ins_encode %{
9438 __ msubw(as_Register($dst$$reg),
9439 as_Register($src1$$reg),
9440 as_Register($src2$$reg),
9441 as_Register($src3$$reg));
9442 %}
9443
9444 ins_pipe(imac_reg_reg);
9445 %}
9446
9447 // Combined Integer Multiply & Neg
9448
9449 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9450 match(Set dst (MulI (SubI zero src1) src2));
9451
9452 ins_cost(INSN_COST * 3);
9453 format %{ "mneg $dst, $src1, $src2" %}
9454
9455 ins_encode %{
9456 __ mnegw(as_Register($dst$$reg),
9457 as_Register($src1$$reg),
9458 as_Register($src2$$reg));
9459 %}
9460
9461 ins_pipe(imac_reg_reg);
9462 %}
9463
9464 // Combined Long Multiply & Add/Sub
9465
9466 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9467 match(Set dst (AddL src3 (MulL src1 src2)));
9468
9469 ins_cost(INSN_COST * 5);
9470 format %{ "madd $dst, $src1, $src2, $src3" %}
9471
9472 ins_encode %{
9473 __ madd(as_Register($dst$$reg),
9474 as_Register($src1$$reg),
9475 as_Register($src2$$reg),
9476 as_Register($src3$$reg));
9477 %}
9478
9479 ins_pipe(lmac_reg_reg);
9480 %}
9481
9482 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9483 match(Set dst (SubL src3 (MulL src1 src2)));
9484
9485 ins_cost(INSN_COST * 5);
9486 format %{ "msub $dst, $src1, $src2, $src3" %}
9487
9488 ins_encode %{
9489 __ msub(as_Register($dst$$reg),
9490 as_Register($src1$$reg),
9491 as_Register($src2$$reg),
9492 as_Register($src3$$reg));
9493 %}
9494
9495 ins_pipe(lmac_reg_reg);
9496 %}
9497
9498 // Combined Long Multiply & Neg
9499
9500 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9501 match(Set dst (MulL (SubL zero src1) src2));
9502
9503 ins_cost(INSN_COST * 5);
9504 format %{ "mneg $dst, $src1, $src2" %}
9505
9506 ins_encode %{
9507 __ mneg(as_Register($dst$$reg),
9508 as_Register($src1$$reg),
9509 as_Register($src2$$reg));
9510 %}
9511
9512 ins_pipe(lmac_reg_reg);
9513 %}
9514
9515 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9516
9517 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9518 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9519
9520 ins_cost(INSN_COST * 3);
9521 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9522
9523 ins_encode %{
9524 __ smaddl(as_Register($dst$$reg),
9525 as_Register($src1$$reg),
9526 as_Register($src2$$reg),
9527 as_Register($src3$$reg));
9528 %}
9529
9530 ins_pipe(imac_reg_reg);
9531 %}
9532
9533 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9534 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9535
9536 ins_cost(INSN_COST * 3);
9537 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9538
9539 ins_encode %{
9540 __ smsubl(as_Register($dst$$reg),
9541 as_Register($src1$$reg),
9542 as_Register($src2$$reg),
9543 as_Register($src3$$reg));
9544 %}
9545
9546 ins_pipe(imac_reg_reg);
9547 %}
9548
9549 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9550 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9551
9552 ins_cost(INSN_COST * 3);
9553 format %{ "smnegl $dst, $src1, $src2" %}
9554
9555 ins_encode %{
9556 __ smnegl(as_Register($dst$$reg),
9557 as_Register($src1$$reg),
9558 as_Register($src2$$reg));
9559 %}
9560
9561 ins_pipe(imac_reg_reg);
9562 %}
9563
9564 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9565
9566 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9567 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9568
9569 ins_cost(INSN_COST * 5);
9570 format %{ "mulw rscratch1, $src1, $src2\n\t"
9571 "maddw $dst, $src3, $src4, rscratch1" %}
9572
9573 ins_encode %{
9574 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9575 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9576
9577 ins_pipe(imac_reg_reg);
9578 %}
9579
9580 // Integer Divide
9581
9582 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9583 match(Set dst (DivI src1 src2));
9584
9585 ins_cost(INSN_COST * 19);
9586 format %{ "sdivw $dst, $src1, $src2" %}
9587
9588 ins_encode(aarch64_enc_divw(dst, src1, src2));
9589 ins_pipe(idiv_reg_reg);
9590 %}
9591
9592 // Long Divide
9593
9594 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9595 match(Set dst (DivL src1 src2));
9596
9597 ins_cost(INSN_COST * 35);
9598 format %{ "sdiv $dst, $src1, $src2" %}
9599
9600 ins_encode(aarch64_enc_div(dst, src1, src2));
9601 ins_pipe(ldiv_reg_reg);
9602 %}
9603
9604 // Integer Remainder
9605
9606 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9607 match(Set dst (ModI src1 src2));
9608
9609 ins_cost(INSN_COST * 22);
9610 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9611 "msubw $dst, rscratch1, $src2, $src1" %}
9612
9613 ins_encode(aarch64_enc_modw(dst, src1, src2));
9614 ins_pipe(idiv_reg_reg);
9615 %}
9616
9617 // Long Remainder
9618
9619 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9620 match(Set dst (ModL src1 src2));
9621
9622 ins_cost(INSN_COST * 38);
9623 format %{ "sdiv rscratch1, $src1, $src2\n"
9624 "msub $dst, rscratch1, $src2, $src1" %}
9625
9626 ins_encode(aarch64_enc_mod(dst, src1, src2));
9627 ins_pipe(ldiv_reg_reg);
9628 %}
9629
9630 // Unsigned Integer Divide
9631
9632 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9633 match(Set dst (UDivI src1 src2));
9634
9635 ins_cost(INSN_COST * 19);
9636 format %{ "udivw $dst, $src1, $src2" %}
9637
9638 ins_encode %{
9639 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9640 %}
9641
9642 ins_pipe(idiv_reg_reg);
9643 %}
9644
9645 // Unsigned Long Divide
9646
9647 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9648 match(Set dst (UDivL src1 src2));
9649
9650 ins_cost(INSN_COST * 35);
9651 format %{ "udiv $dst, $src1, $src2" %}
9652
9653 ins_encode %{
9654 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9655 %}
9656
9657 ins_pipe(ldiv_reg_reg);
9658 %}
9659
9660 // Unsigned Integer Remainder
9661
9662 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9663 match(Set dst (UModI src1 src2));
9664
9665 ins_cost(INSN_COST * 22);
9666 format %{ "udivw rscratch1, $src1, $src2\n\t"
9667 "msubw $dst, rscratch1, $src2, $src1" %}
9668
9669 ins_encode %{
9670 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9671 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9672 %}
9673
9674 ins_pipe(idiv_reg_reg);
9675 %}
9676
9677 // Unsigned Long Remainder
9678
9679 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9680 match(Set dst (UModL src1 src2));
9681
9682 ins_cost(INSN_COST * 38);
9683 format %{ "udiv rscratch1, $src1, $src2\n"
9684 "msub $dst, rscratch1, $src2, $src1" %}
9685
9686 ins_encode %{
9687 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9688 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9689 %}
9690
9691 ins_pipe(ldiv_reg_reg);
9692 %}
9693
9694 // Integer Shifts
9695
9696 // Shift Left Register
9697 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9698 match(Set dst (LShiftI src1 src2));
9699
9700 ins_cost(INSN_COST * 2);
9701 format %{ "lslvw $dst, $src1, $src2" %}
9702
9703 ins_encode %{
9704 __ lslvw(as_Register($dst$$reg),
9705 as_Register($src1$$reg),
9706 as_Register($src2$$reg));
9707 %}
9708
9709 ins_pipe(ialu_reg_reg_vshift);
9710 %}
9711
9712 // Shift Left Immediate
9713 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9714 match(Set dst (LShiftI src1 src2));
9715
9716 ins_cost(INSN_COST);
9717 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9718
9719 ins_encode %{
9720 __ lslw(as_Register($dst$$reg),
9721 as_Register($src1$$reg),
9722 $src2$$constant & 0x1f);
9723 %}
9724
9725 ins_pipe(ialu_reg_shift);
9726 %}
9727
9728 // Shift Right Logical Register
9729 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9730 match(Set dst (URShiftI src1 src2));
9731
9732 ins_cost(INSN_COST * 2);
9733 format %{ "lsrvw $dst, $src1, $src2" %}
9734
9735 ins_encode %{
9736 __ lsrvw(as_Register($dst$$reg),
9737 as_Register($src1$$reg),
9738 as_Register($src2$$reg));
9739 %}
9740
9741 ins_pipe(ialu_reg_reg_vshift);
9742 %}
9743
9744 // Shift Right Logical Immediate
9745 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9746 match(Set dst (URShiftI src1 src2));
9747
9748 ins_cost(INSN_COST);
9749 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9750
9751 ins_encode %{
9752 __ lsrw(as_Register($dst$$reg),
9753 as_Register($src1$$reg),
9754 $src2$$constant & 0x1f);
9755 %}
9756
9757 ins_pipe(ialu_reg_shift);
9758 %}
9759
9760 // Shift Right Arithmetic Register
9761 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9762 match(Set dst (RShiftI src1 src2));
9763
9764 ins_cost(INSN_COST * 2);
9765 format %{ "asrvw $dst, $src1, $src2" %}
9766
9767 ins_encode %{
9768 __ asrvw(as_Register($dst$$reg),
9769 as_Register($src1$$reg),
9770 as_Register($src2$$reg));
9771 %}
9772
9773 ins_pipe(ialu_reg_reg_vshift);
9774 %}
9775
9776 // Shift Right Arithmetic Immediate
9777 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9778 match(Set dst (RShiftI src1 src2));
9779
9780 ins_cost(INSN_COST);
9781 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9782
9783 ins_encode %{
9784 __ asrw(as_Register($dst$$reg),
9785 as_Register($src1$$reg),
9786 $src2$$constant & 0x1f);
9787 %}
9788
9789 ins_pipe(ialu_reg_shift);
9790 %}
9791
9792 // Combined Int Mask and Right Shift (using UBFM)
9793 // TODO
9794
9795 // Long Shifts
9796
9797 // Shift Left Register
9798 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9799 match(Set dst (LShiftL src1 src2));
9800
9801 ins_cost(INSN_COST * 2);
9802 format %{ "lslv $dst, $src1, $src2" %}
9803
9804 ins_encode %{
9805 __ lslv(as_Register($dst$$reg),
9806 as_Register($src1$$reg),
9807 as_Register($src2$$reg));
9808 %}
9809
9810 ins_pipe(ialu_reg_reg_vshift);
9811 %}
9812
9813 // Shift Left Immediate
9814 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9815 match(Set dst (LShiftL src1 src2));
9816
9817 ins_cost(INSN_COST);
9818 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9819
9820 ins_encode %{
9821 __ lsl(as_Register($dst$$reg),
9822 as_Register($src1$$reg),
9823 $src2$$constant & 0x3f);
9824 %}
9825
9826 ins_pipe(ialu_reg_shift);
9827 %}
9828
9829 // Shift Right Logical Register
9830 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9831 match(Set dst (URShiftL src1 src2));
9832
9833 ins_cost(INSN_COST * 2);
9834 format %{ "lsrv $dst, $src1, $src2" %}
9835
9836 ins_encode %{
9837 __ lsrv(as_Register($dst$$reg),
9838 as_Register($src1$$reg),
9839 as_Register($src2$$reg));
9840 %}
9841
9842 ins_pipe(ialu_reg_reg_vshift);
9843 %}
9844
9845 // Shift Right Logical Immediate
9846 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9847 match(Set dst (URShiftL src1 src2));
9848
9849 ins_cost(INSN_COST);
9850 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9851
9852 ins_encode %{
9853 __ lsr(as_Register($dst$$reg),
9854 as_Register($src1$$reg),
9855 $src2$$constant & 0x3f);
9856 %}
9857
9858 ins_pipe(ialu_reg_shift);
9859 %}
9860
9861 // A special-case pattern for card table stores.
9862 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9863 match(Set dst (URShiftL (CastP2X src1) src2));
9864
9865 ins_cost(INSN_COST);
9866 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9867
9868 ins_encode %{
9869 __ lsr(as_Register($dst$$reg),
9870 as_Register($src1$$reg),
9871 $src2$$constant & 0x3f);
9872 %}
9873
9874 ins_pipe(ialu_reg_shift);
9875 %}
9876
9877 // Shift Right Arithmetic Register
9878 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9879 match(Set dst (RShiftL src1 src2));
9880
9881 ins_cost(INSN_COST * 2);
9882 format %{ "asrv $dst, $src1, $src2" %}
9883
9884 ins_encode %{
9885 __ asrv(as_Register($dst$$reg),
9886 as_Register($src1$$reg),
9887 as_Register($src2$$reg));
9888 %}
9889
9890 ins_pipe(ialu_reg_reg_vshift);
9891 %}
9892
9893 // Shift Right Arithmetic Immediate
9894 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9895 match(Set dst (RShiftL src1 src2));
9896
9897 ins_cost(INSN_COST);
9898 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9899
9900 ins_encode %{
9901 __ asr(as_Register($dst$$reg),
9902 as_Register($src1$$reg),
9903 $src2$$constant & 0x3f);
9904 %}
9905
9906 ins_pipe(ialu_reg_shift);
9907 %}
9908
9909 // BEGIN This section of the file is automatically generated. Do not edit --------------
9910 // This section is generated from aarch64_ad.m4
9911
9912 // This pattern is automatically generated from aarch64_ad.m4
9913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9914 instruct regL_not_reg(iRegLNoSp dst,
9915 iRegL src1, immL_M1 m1,
9916 rFlagsReg cr) %{
9917 match(Set dst (XorL src1 m1));
9918 ins_cost(INSN_COST);
9919 format %{ "eon $dst, $src1, zr" %}
9920
9921 ins_encode %{
9922 __ eon(as_Register($dst$$reg),
9923 as_Register($src1$$reg),
9924 zr,
9925 Assembler::LSL, 0);
9926 %}
9927
9928 ins_pipe(ialu_reg);
9929 %}
9930
9931 // This pattern is automatically generated from aarch64_ad.m4
9932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9933 instruct regI_not_reg(iRegINoSp dst,
9934 iRegIorL2I src1, immI_M1 m1,
9935 rFlagsReg cr) %{
9936 match(Set dst (XorI src1 m1));
9937 ins_cost(INSN_COST);
9938 format %{ "eonw $dst, $src1, zr" %}
9939
9940 ins_encode %{
9941 __ eonw(as_Register($dst$$reg),
9942 as_Register($src1$$reg),
9943 zr,
9944 Assembler::LSL, 0);
9945 %}
9946
9947 ins_pipe(ialu_reg);
9948 %}
9949
9950 // This pattern is automatically generated from aarch64_ad.m4
9951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9952 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9953 immI0 zero, iRegIorL2I src1, immI src2) %{
9954 match(Set dst (SubI zero (URShiftI src1 src2)));
9955
9956 ins_cost(1.9 * INSN_COST);
9957 format %{ "negw $dst, $src1, LSR $src2" %}
9958
9959 ins_encode %{
9960 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9961 Assembler::LSR, $src2$$constant & 0x1f);
9962 %}
9963
9964 ins_pipe(ialu_reg_shift);
9965 %}
9966
9967 // This pattern is automatically generated from aarch64_ad.m4
9968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9969 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9970 immI0 zero, iRegIorL2I src1, immI src2) %{
9971 match(Set dst (SubI zero (RShiftI src1 src2)));
9972
9973 ins_cost(1.9 * INSN_COST);
9974 format %{ "negw $dst, $src1, ASR $src2" %}
9975
9976 ins_encode %{
9977 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9978 Assembler::ASR, $src2$$constant & 0x1f);
9979 %}
9980
9981 ins_pipe(ialu_reg_shift);
9982 %}
9983
9984 // This pattern is automatically generated from aarch64_ad.m4
9985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9986 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9987 immI0 zero, iRegIorL2I src1, immI src2) %{
9988 match(Set dst (SubI zero (LShiftI src1 src2)));
9989
9990 ins_cost(1.9 * INSN_COST);
9991 format %{ "negw $dst, $src1, LSL $src2" %}
9992
9993 ins_encode %{
9994 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9995 Assembler::LSL, $src2$$constant & 0x1f);
9996 %}
9997
9998 ins_pipe(ialu_reg_shift);
9999 %}
10000
10001 // This pattern is automatically generated from aarch64_ad.m4
10002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10003 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10004 immL0 zero, iRegL src1, immI src2) %{
10005 match(Set dst (SubL zero (URShiftL src1 src2)));
10006
10007 ins_cost(1.9 * INSN_COST);
10008 format %{ "neg $dst, $src1, LSR $src2" %}
10009
10010 ins_encode %{
10011 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10012 Assembler::LSR, $src2$$constant & 0x3f);
10013 %}
10014
10015 ins_pipe(ialu_reg_shift);
10016 %}
10017
10018 // This pattern is automatically generated from aarch64_ad.m4
10019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10020 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10021 immL0 zero, iRegL src1, immI src2) %{
10022 match(Set dst (SubL zero (RShiftL src1 src2)));
10023
10024 ins_cost(1.9 * INSN_COST);
10025 format %{ "neg $dst, $src1, ASR $src2" %}
10026
10027 ins_encode %{
10028 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10029 Assembler::ASR, $src2$$constant & 0x3f);
10030 %}
10031
10032 ins_pipe(ialu_reg_shift);
10033 %}
10034
10035 // This pattern is automatically generated from aarch64_ad.m4
10036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10037 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10038 immL0 zero, iRegL src1, immI src2) %{
10039 match(Set dst (SubL zero (LShiftL src1 src2)));
10040
10041 ins_cost(1.9 * INSN_COST);
10042 format %{ "neg $dst, $src1, LSL $src2" %}
10043
10044 ins_encode %{
10045 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10046 Assembler::LSL, $src2$$constant & 0x3f);
10047 %}
10048
10049 ins_pipe(ialu_reg_shift);
10050 %}
10051
10052 // This pattern is automatically generated from aarch64_ad.m4
10053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10054 instruct AndI_reg_not_reg(iRegINoSp dst,
10055 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10056 match(Set dst (AndI src1 (XorI src2 m1)));
10057 ins_cost(INSN_COST);
10058 format %{ "bicw $dst, $src1, $src2" %}
10059
10060 ins_encode %{
10061 __ bicw(as_Register($dst$$reg),
10062 as_Register($src1$$reg),
10063 as_Register($src2$$reg),
10064 Assembler::LSL, 0);
10065 %}
10066
10067 ins_pipe(ialu_reg_reg);
10068 %}
10069
10070 // This pattern is automatically generated from aarch64_ad.m4
10071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10072 instruct AndL_reg_not_reg(iRegLNoSp dst,
10073 iRegL src1, iRegL src2, immL_M1 m1) %{
10074 match(Set dst (AndL src1 (XorL src2 m1)));
10075 ins_cost(INSN_COST);
10076 format %{ "bic $dst, $src1, $src2" %}
10077
10078 ins_encode %{
10079 __ bic(as_Register($dst$$reg),
10080 as_Register($src1$$reg),
10081 as_Register($src2$$reg),
10082 Assembler::LSL, 0);
10083 %}
10084
10085 ins_pipe(ialu_reg_reg);
10086 %}
10087
10088 // This pattern is automatically generated from aarch64_ad.m4
10089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10090 instruct OrI_reg_not_reg(iRegINoSp dst,
10091 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10092 match(Set dst (OrI src1 (XorI src2 m1)));
10093 ins_cost(INSN_COST);
10094 format %{ "ornw $dst, $src1, $src2" %}
10095
10096 ins_encode %{
10097 __ ornw(as_Register($dst$$reg),
10098 as_Register($src1$$reg),
10099 as_Register($src2$$reg),
10100 Assembler::LSL, 0);
10101 %}
10102
10103 ins_pipe(ialu_reg_reg);
10104 %}
10105
10106 // This pattern is automatically generated from aarch64_ad.m4
10107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10108 instruct OrL_reg_not_reg(iRegLNoSp dst,
10109 iRegL src1, iRegL src2, immL_M1 m1) %{
10110 match(Set dst (OrL src1 (XorL src2 m1)));
10111 ins_cost(INSN_COST);
10112 format %{ "orn $dst, $src1, $src2" %}
10113
10114 ins_encode %{
10115 __ orn(as_Register($dst$$reg),
10116 as_Register($src1$$reg),
10117 as_Register($src2$$reg),
10118 Assembler::LSL, 0);
10119 %}
10120
10121 ins_pipe(ialu_reg_reg);
10122 %}
10123
10124 // This pattern is automatically generated from aarch64_ad.m4
10125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10126 instruct XorI_reg_not_reg(iRegINoSp dst,
10127 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10128 match(Set dst (XorI m1 (XorI src2 src1)));
10129 ins_cost(INSN_COST);
10130 format %{ "eonw $dst, $src1, $src2" %}
10131
10132 ins_encode %{
10133 __ eonw(as_Register($dst$$reg),
10134 as_Register($src1$$reg),
10135 as_Register($src2$$reg),
10136 Assembler::LSL, 0);
10137 %}
10138
10139 ins_pipe(ialu_reg_reg);
10140 %}
10141
10142 // This pattern is automatically generated from aarch64_ad.m4
10143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10144 instruct XorL_reg_not_reg(iRegLNoSp dst,
10145 iRegL src1, iRegL src2, immL_M1 m1) %{
10146 match(Set dst (XorL m1 (XorL src2 src1)));
10147 ins_cost(INSN_COST);
10148 format %{ "eon $dst, $src1, $src2" %}
10149
10150 ins_encode %{
10151 __ eon(as_Register($dst$$reg),
10152 as_Register($src1$$reg),
10153 as_Register($src2$$reg),
10154 Assembler::LSL, 0);
10155 %}
10156
10157 ins_pipe(ialu_reg_reg);
10158 %}
10159
10160 // This pattern is automatically generated from aarch64_ad.m4
10161 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10162 // val & (-1 ^ (val >>> shift)) ==> bicw
10163 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10164 iRegIorL2I src1, iRegIorL2I src2,
10165 immI src3, immI_M1 src4) %{
10166 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10167 ins_cost(1.9 * INSN_COST);
10168 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10169
10170 ins_encode %{
10171 __ bicw(as_Register($dst$$reg),
10172 as_Register($src1$$reg),
10173 as_Register($src2$$reg),
10174 Assembler::LSR,
10175 $src3$$constant & 0x1f);
10176 %}
10177
10178 ins_pipe(ialu_reg_reg_shift);
10179 %}
10180
10181 // This pattern is automatically generated from aarch64_ad.m4
10182 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10183 // val & (-1 ^ (val >>> shift)) ==> bic
10184 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10185 iRegL src1, iRegL src2,
10186 immI src3, immL_M1 src4) %{
10187 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10188 ins_cost(1.9 * INSN_COST);
10189 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10190
10191 ins_encode %{
10192 __ bic(as_Register($dst$$reg),
10193 as_Register($src1$$reg),
10194 as_Register($src2$$reg),
10195 Assembler::LSR,
10196 $src3$$constant & 0x3f);
10197 %}
10198
10199 ins_pipe(ialu_reg_reg_shift);
10200 %}
10201
10202 // This pattern is automatically generated from aarch64_ad.m4
10203 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10204 // val & (-1 ^ (val >> shift)) ==> bicw
10205 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10206 iRegIorL2I src1, iRegIorL2I src2,
10207 immI src3, immI_M1 src4) %{
10208 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10209 ins_cost(1.9 * INSN_COST);
10210 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10211
10212 ins_encode %{
10213 __ bicw(as_Register($dst$$reg),
10214 as_Register($src1$$reg),
10215 as_Register($src2$$reg),
10216 Assembler::ASR,
10217 $src3$$constant & 0x1f);
10218 %}
10219
10220 ins_pipe(ialu_reg_reg_shift);
10221 %}
10222
10223 // This pattern is automatically generated from aarch64_ad.m4
10224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10225 // val & (-1 ^ (val >> shift)) ==> bic
10226 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10227 iRegL src1, iRegL src2,
10228 immI src3, immL_M1 src4) %{
10229 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10230 ins_cost(1.9 * INSN_COST);
10231 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10232
10233 ins_encode %{
10234 __ bic(as_Register($dst$$reg),
10235 as_Register($src1$$reg),
10236 as_Register($src2$$reg),
10237 Assembler::ASR,
10238 $src3$$constant & 0x3f);
10239 %}
10240
10241 ins_pipe(ialu_reg_reg_shift);
10242 %}
10243
10244 // This pattern is automatically generated from aarch64_ad.m4
10245 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10246 // val & (-1 ^ (val ror shift)) ==> bicw
10247 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10248 iRegIorL2I src1, iRegIorL2I src2,
10249 immI src3, immI_M1 src4) %{
10250 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10251 ins_cost(1.9 * INSN_COST);
10252 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10253
10254 ins_encode %{
10255 __ bicw(as_Register($dst$$reg),
10256 as_Register($src1$$reg),
10257 as_Register($src2$$reg),
10258 Assembler::ROR,
10259 $src3$$constant & 0x1f);
10260 %}
10261
10262 ins_pipe(ialu_reg_reg_shift);
10263 %}
10264
10265 // This pattern is automatically generated from aarch64_ad.m4
10266 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10267 // val & (-1 ^ (val ror shift)) ==> bic
10268 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10269 iRegL src1, iRegL src2,
10270 immI src3, immL_M1 src4) %{
10271 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10272 ins_cost(1.9 * INSN_COST);
10273 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10274
10275 ins_encode %{
10276 __ bic(as_Register($dst$$reg),
10277 as_Register($src1$$reg),
10278 as_Register($src2$$reg),
10279 Assembler::ROR,
10280 $src3$$constant & 0x3f);
10281 %}
10282
10283 ins_pipe(ialu_reg_reg_shift);
10284 %}
10285
10286 // This pattern is automatically generated from aarch64_ad.m4
10287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10288 // val & (-1 ^ (val << shift)) ==> bicw
10289 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10290 iRegIorL2I src1, iRegIorL2I src2,
10291 immI src3, immI_M1 src4) %{
10292 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10293 ins_cost(1.9 * INSN_COST);
10294 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10295
10296 ins_encode %{
10297 __ bicw(as_Register($dst$$reg),
10298 as_Register($src1$$reg),
10299 as_Register($src2$$reg),
10300 Assembler::LSL,
10301 $src3$$constant & 0x1f);
10302 %}
10303
10304 ins_pipe(ialu_reg_reg_shift);
10305 %}
10306
10307 // This pattern is automatically generated from aarch64_ad.m4
10308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10309 // val & (-1 ^ (val << shift)) ==> bic
10310 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10311 iRegL src1, iRegL src2,
10312 immI src3, immL_M1 src4) %{
10313 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10314 ins_cost(1.9 * INSN_COST);
10315 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10316
10317 ins_encode %{
10318 __ bic(as_Register($dst$$reg),
10319 as_Register($src1$$reg),
10320 as_Register($src2$$reg),
10321 Assembler::LSL,
10322 $src3$$constant & 0x3f);
10323 %}
10324
10325 ins_pipe(ialu_reg_reg_shift);
10326 %}
10327
10328 // This pattern is automatically generated from aarch64_ad.m4
10329 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10330 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10331 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10332 iRegIorL2I src1, iRegIorL2I src2,
10333 immI src3, immI_M1 src4) %{
10334 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10335 ins_cost(1.9 * INSN_COST);
10336 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10337
10338 ins_encode %{
10339 __ eonw(as_Register($dst$$reg),
10340 as_Register($src1$$reg),
10341 as_Register($src2$$reg),
10342 Assembler::LSR,
10343 $src3$$constant & 0x1f);
10344 %}
10345
10346 ins_pipe(ialu_reg_reg_shift);
10347 %}
10348
10349 // This pattern is automatically generated from aarch64_ad.m4
10350 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10351 // val ^ (-1 ^ (val >>> shift)) ==> eon
10352 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10353 iRegL src1, iRegL src2,
10354 immI src3, immL_M1 src4) %{
10355 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10356 ins_cost(1.9 * INSN_COST);
10357 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10358
10359 ins_encode %{
10360 __ eon(as_Register($dst$$reg),
10361 as_Register($src1$$reg),
10362 as_Register($src2$$reg),
10363 Assembler::LSR,
10364 $src3$$constant & 0x3f);
10365 %}
10366
10367 ins_pipe(ialu_reg_reg_shift);
10368 %}
10369
10370 // This pattern is automatically generated from aarch64_ad.m4
10371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10372 // val ^ (-1 ^ (val >> shift)) ==> eonw
10373 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10374 iRegIorL2I src1, iRegIorL2I src2,
10375 immI src3, immI_M1 src4) %{
10376 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10377 ins_cost(1.9 * INSN_COST);
10378 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10379
10380 ins_encode %{
10381 __ eonw(as_Register($dst$$reg),
10382 as_Register($src1$$reg),
10383 as_Register($src2$$reg),
10384 Assembler::ASR,
10385 $src3$$constant & 0x1f);
10386 %}
10387
10388 ins_pipe(ialu_reg_reg_shift);
10389 %}
10390
10391 // This pattern is automatically generated from aarch64_ad.m4
10392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10393 // val ^ (-1 ^ (val >> shift)) ==> eon
10394 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10395 iRegL src1, iRegL src2,
10396 immI src3, immL_M1 src4) %{
10397 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10398 ins_cost(1.9 * INSN_COST);
10399 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10400
10401 ins_encode %{
10402 __ eon(as_Register($dst$$reg),
10403 as_Register($src1$$reg),
10404 as_Register($src2$$reg),
10405 Assembler::ASR,
10406 $src3$$constant & 0x3f);
10407 %}
10408
10409 ins_pipe(ialu_reg_reg_shift);
10410 %}
10411
10412 // This pattern is automatically generated from aarch64_ad.m4
10413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10414 // val ^ (-1 ^ (val ror shift)) ==> eonw
10415 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10416 iRegIorL2I src1, iRegIorL2I src2,
10417 immI src3, immI_M1 src4) %{
10418 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10419 ins_cost(1.9 * INSN_COST);
10420 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10421
10422 ins_encode %{
10423 __ eonw(as_Register($dst$$reg),
10424 as_Register($src1$$reg),
10425 as_Register($src2$$reg),
10426 Assembler::ROR,
10427 $src3$$constant & 0x1f);
10428 %}
10429
10430 ins_pipe(ialu_reg_reg_shift);
10431 %}
10432
10433 // This pattern is automatically generated from aarch64_ad.m4
10434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10435 // val ^ (-1 ^ (val ror shift)) ==> eon
10436 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10437 iRegL src1, iRegL src2,
10438 immI src3, immL_M1 src4) %{
10439 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10440 ins_cost(1.9 * INSN_COST);
10441 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10442
10443 ins_encode %{
10444 __ eon(as_Register($dst$$reg),
10445 as_Register($src1$$reg),
10446 as_Register($src2$$reg),
10447 Assembler::ROR,
10448 $src3$$constant & 0x3f);
10449 %}
10450
10451 ins_pipe(ialu_reg_reg_shift);
10452 %}
10453
10454 // This pattern is automatically generated from aarch64_ad.m4
10455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10456 // val ^ (-1 ^ (val << shift)) ==> eonw
10457 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10458 iRegIorL2I src1, iRegIorL2I src2,
10459 immI src3, immI_M1 src4) %{
10460 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10461 ins_cost(1.9 * INSN_COST);
10462 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10463
10464 ins_encode %{
10465 __ eonw(as_Register($dst$$reg),
10466 as_Register($src1$$reg),
10467 as_Register($src2$$reg),
10468 Assembler::LSL,
10469 $src3$$constant & 0x1f);
10470 %}
10471
10472 ins_pipe(ialu_reg_reg_shift);
10473 %}
10474
10475 // This pattern is automatically generated from aarch64_ad.m4
10476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10477 // val ^ (-1 ^ (val << shift)) ==> eon
10478 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10479 iRegL src1, iRegL src2,
10480 immI src3, immL_M1 src4) %{
10481 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10482 ins_cost(1.9 * INSN_COST);
10483 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10484
10485 ins_encode %{
10486 __ eon(as_Register($dst$$reg),
10487 as_Register($src1$$reg),
10488 as_Register($src2$$reg),
10489 Assembler::LSL,
10490 $src3$$constant & 0x3f);
10491 %}
10492
10493 ins_pipe(ialu_reg_reg_shift);
10494 %}
10495
10496 // This pattern is automatically generated from aarch64_ad.m4
10497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10498 // val | (-1 ^ (val >>> shift)) ==> ornw
10499 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10500 iRegIorL2I src1, iRegIorL2I src2,
10501 immI src3, immI_M1 src4) %{
10502 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10503 ins_cost(1.9 * INSN_COST);
10504 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10505
10506 ins_encode %{
10507 __ ornw(as_Register($dst$$reg),
10508 as_Register($src1$$reg),
10509 as_Register($src2$$reg),
10510 Assembler::LSR,
10511 $src3$$constant & 0x1f);
10512 %}
10513
10514 ins_pipe(ialu_reg_reg_shift);
10515 %}
10516
10517 // This pattern is automatically generated from aarch64_ad.m4
10518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10519 // val | (-1 ^ (val >>> shift)) ==> orn
10520 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10521 iRegL src1, iRegL src2,
10522 immI src3, immL_M1 src4) %{
10523 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10524 ins_cost(1.9 * INSN_COST);
10525 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10526
10527 ins_encode %{
10528 __ orn(as_Register($dst$$reg),
10529 as_Register($src1$$reg),
10530 as_Register($src2$$reg),
10531 Assembler::LSR,
10532 $src3$$constant & 0x3f);
10533 %}
10534
10535 ins_pipe(ialu_reg_reg_shift);
10536 %}
10537
10538 // This pattern is automatically generated from aarch64_ad.m4
10539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10540 // val | (-1 ^ (val >> shift)) ==> ornw
10541 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10542 iRegIorL2I src1, iRegIorL2I src2,
10543 immI src3, immI_M1 src4) %{
10544 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10545 ins_cost(1.9 * INSN_COST);
10546 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10547
10548 ins_encode %{
10549 __ ornw(as_Register($dst$$reg),
10550 as_Register($src1$$reg),
10551 as_Register($src2$$reg),
10552 Assembler::ASR,
10553 $src3$$constant & 0x1f);
10554 %}
10555
10556 ins_pipe(ialu_reg_reg_shift);
10557 %}
10558
10559 // This pattern is automatically generated from aarch64_ad.m4
10560 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10561 // val | (-1 ^ (val >> shift)) ==> orn
10562 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10563 iRegL src1, iRegL src2,
10564 immI src3, immL_M1 src4) %{
10565 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10566 ins_cost(1.9 * INSN_COST);
10567 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10568
10569 ins_encode %{
10570 __ orn(as_Register($dst$$reg),
10571 as_Register($src1$$reg),
10572 as_Register($src2$$reg),
10573 Assembler::ASR,
10574 $src3$$constant & 0x3f);
10575 %}
10576
10577 ins_pipe(ialu_reg_reg_shift);
10578 %}
10579
10580 // This pattern is automatically generated from aarch64_ad.m4
10581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10582 // val | (-1 ^ (val ror shift)) ==> ornw
10583 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10584 iRegIorL2I src1, iRegIorL2I src2,
10585 immI src3, immI_M1 src4) %{
10586 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10587 ins_cost(1.9 * INSN_COST);
10588 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10589
10590 ins_encode %{
10591 __ ornw(as_Register($dst$$reg),
10592 as_Register($src1$$reg),
10593 as_Register($src2$$reg),
10594 Assembler::ROR,
10595 $src3$$constant & 0x1f);
10596 %}
10597
10598 ins_pipe(ialu_reg_reg_shift);
10599 %}
10600
10601 // This pattern is automatically generated from aarch64_ad.m4
10602 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10603 // val | (-1 ^ (val ror shift)) ==> orn
10604 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10605 iRegL src1, iRegL src2,
10606 immI src3, immL_M1 src4) %{
10607 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10608 ins_cost(1.9 * INSN_COST);
10609 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10610
10611 ins_encode %{
10612 __ orn(as_Register($dst$$reg),
10613 as_Register($src1$$reg),
10614 as_Register($src2$$reg),
10615 Assembler::ROR,
10616 $src3$$constant & 0x3f);
10617 %}
10618
10619 ins_pipe(ialu_reg_reg_shift);
10620 %}
10621
10622 // This pattern is automatically generated from aarch64_ad.m4
10623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10624 // val | (-1 ^ (val << shift)) ==> ornw
10625 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10626 iRegIorL2I src1, iRegIorL2I src2,
10627 immI src3, immI_M1 src4) %{
10628 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10629 ins_cost(1.9 * INSN_COST);
10630 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10631
10632 ins_encode %{
10633 __ ornw(as_Register($dst$$reg),
10634 as_Register($src1$$reg),
10635 as_Register($src2$$reg),
10636 Assembler::LSL,
10637 $src3$$constant & 0x1f);
10638 %}
10639
10640 ins_pipe(ialu_reg_reg_shift);
10641 %}
10642
10643 // This pattern is automatically generated from aarch64_ad.m4
10644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10645 // val | (-1 ^ (val << shift)) ==> orn
10646 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10647 iRegL src1, iRegL src2,
10648 immI src3, immL_M1 src4) %{
10649 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10650 ins_cost(1.9 * INSN_COST);
10651 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10652
10653 ins_encode %{
10654 __ orn(as_Register($dst$$reg),
10655 as_Register($src1$$reg),
10656 as_Register($src2$$reg),
10657 Assembler::LSL,
10658 $src3$$constant & 0x3f);
10659 %}
10660
10661 ins_pipe(ialu_reg_reg_shift);
10662 %}
10663
10664 // This pattern is automatically generated from aarch64_ad.m4
10665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10666 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10667 iRegIorL2I src1, iRegIorL2I src2,
10668 immI src3) %{
10669 match(Set dst (AndI src1 (URShiftI src2 src3)));
10670
10671 ins_cost(1.9 * INSN_COST);
10672 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10673
10674 ins_encode %{
10675 __ andw(as_Register($dst$$reg),
10676 as_Register($src1$$reg),
10677 as_Register($src2$$reg),
10678 Assembler::LSR,
10679 $src3$$constant & 0x1f);
10680 %}
10681
10682 ins_pipe(ialu_reg_reg_shift);
10683 %}
10684
10685 // This pattern is automatically generated from aarch64_ad.m4
10686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10687 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10688 iRegL src1, iRegL src2,
10689 immI src3) %{
10690 match(Set dst (AndL src1 (URShiftL src2 src3)));
10691
10692 ins_cost(1.9 * INSN_COST);
10693 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10694
10695 ins_encode %{
10696 __ andr(as_Register($dst$$reg),
10697 as_Register($src1$$reg),
10698 as_Register($src2$$reg),
10699 Assembler::LSR,
10700 $src3$$constant & 0x3f);
10701 %}
10702
10703 ins_pipe(ialu_reg_reg_shift);
10704 %}
10705
10706 // This pattern is automatically generated from aarch64_ad.m4
10707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10708 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10709 iRegIorL2I src1, iRegIorL2I src2,
10710 immI src3) %{
10711 match(Set dst (AndI src1 (RShiftI src2 src3)));
10712
10713 ins_cost(1.9 * INSN_COST);
10714 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10715
10716 ins_encode %{
10717 __ andw(as_Register($dst$$reg),
10718 as_Register($src1$$reg),
10719 as_Register($src2$$reg),
10720 Assembler::ASR,
10721 $src3$$constant & 0x1f);
10722 %}
10723
10724 ins_pipe(ialu_reg_reg_shift);
10725 %}
10726
10727 // This pattern is automatically generated from aarch64_ad.m4
10728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10729 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10730 iRegL src1, iRegL src2,
10731 immI src3) %{
10732 match(Set dst (AndL src1 (RShiftL src2 src3)));
10733
10734 ins_cost(1.9 * INSN_COST);
10735 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10736
10737 ins_encode %{
10738 __ andr(as_Register($dst$$reg),
10739 as_Register($src1$$reg),
10740 as_Register($src2$$reg),
10741 Assembler::ASR,
10742 $src3$$constant & 0x3f);
10743 %}
10744
10745 ins_pipe(ialu_reg_reg_shift);
10746 %}
10747
10748 // This pattern is automatically generated from aarch64_ad.m4
10749 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10750 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10751 iRegIorL2I src1, iRegIorL2I src2,
10752 immI src3) %{
10753 match(Set dst (AndI src1 (LShiftI src2 src3)));
10754
10755 ins_cost(1.9 * INSN_COST);
10756 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10757
10758 ins_encode %{
10759 __ andw(as_Register($dst$$reg),
10760 as_Register($src1$$reg),
10761 as_Register($src2$$reg),
10762 Assembler::LSL,
10763 $src3$$constant & 0x1f);
10764 %}
10765
10766 ins_pipe(ialu_reg_reg_shift);
10767 %}
10768
10769 // This pattern is automatically generated from aarch64_ad.m4
10770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10771 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10772 iRegL src1, iRegL src2,
10773 immI src3) %{
10774 match(Set dst (AndL src1 (LShiftL src2 src3)));
10775
10776 ins_cost(1.9 * INSN_COST);
10777 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10778
10779 ins_encode %{
10780 __ andr(as_Register($dst$$reg),
10781 as_Register($src1$$reg),
10782 as_Register($src2$$reg),
10783 Assembler::LSL,
10784 $src3$$constant & 0x3f);
10785 %}
10786
10787 ins_pipe(ialu_reg_reg_shift);
10788 %}
10789
10790 // This pattern is automatically generated from aarch64_ad.m4
10791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10792 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10793 iRegIorL2I src1, iRegIorL2I src2,
10794 immI src3) %{
10795 match(Set dst (AndI src1 (RotateRight src2 src3)));
10796
10797 ins_cost(1.9 * INSN_COST);
10798 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10799
10800 ins_encode %{
10801 __ andw(as_Register($dst$$reg),
10802 as_Register($src1$$reg),
10803 as_Register($src2$$reg),
10804 Assembler::ROR,
10805 $src3$$constant & 0x1f);
10806 %}
10807
10808 ins_pipe(ialu_reg_reg_shift);
10809 %}
10810
10811 // This pattern is automatically generated from aarch64_ad.m4
10812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10813 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10814 iRegL src1, iRegL src2,
10815 immI src3) %{
10816 match(Set dst (AndL src1 (RotateRight src2 src3)));
10817
10818 ins_cost(1.9 * INSN_COST);
10819 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10820
10821 ins_encode %{
10822 __ andr(as_Register($dst$$reg),
10823 as_Register($src1$$reg),
10824 as_Register($src2$$reg),
10825 Assembler::ROR,
10826 $src3$$constant & 0x3f);
10827 %}
10828
10829 ins_pipe(ialu_reg_reg_shift);
10830 %}
10831
10832 // This pattern is automatically generated from aarch64_ad.m4
10833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10834 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10835 iRegIorL2I src1, iRegIorL2I src2,
10836 immI src3) %{
10837 match(Set dst (XorI src1 (URShiftI src2 src3)));
10838
10839 ins_cost(1.9 * INSN_COST);
10840 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10841
10842 ins_encode %{
10843 __ eorw(as_Register($dst$$reg),
10844 as_Register($src1$$reg),
10845 as_Register($src2$$reg),
10846 Assembler::LSR,
10847 $src3$$constant & 0x1f);
10848 %}
10849
10850 ins_pipe(ialu_reg_reg_shift);
10851 %}
10852
10853 // This pattern is automatically generated from aarch64_ad.m4
10854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10855 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10856 iRegL src1, iRegL src2,
10857 immI src3) %{
10858 match(Set dst (XorL src1 (URShiftL src2 src3)));
10859
10860 ins_cost(1.9 * INSN_COST);
10861 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10862
10863 ins_encode %{
10864 __ eor(as_Register($dst$$reg),
10865 as_Register($src1$$reg),
10866 as_Register($src2$$reg),
10867 Assembler::LSR,
10868 $src3$$constant & 0x3f);
10869 %}
10870
10871 ins_pipe(ialu_reg_reg_shift);
10872 %}
10873
10874 // This pattern is automatically generated from aarch64_ad.m4
10875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10876 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10877 iRegIorL2I src1, iRegIorL2I src2,
10878 immI src3) %{
10879 match(Set dst (XorI src1 (RShiftI src2 src3)));
10880
10881 ins_cost(1.9 * INSN_COST);
10882 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10883
10884 ins_encode %{
10885 __ eorw(as_Register($dst$$reg),
10886 as_Register($src1$$reg),
10887 as_Register($src2$$reg),
10888 Assembler::ASR,
10889 $src3$$constant & 0x1f);
10890 %}
10891
10892 ins_pipe(ialu_reg_reg_shift);
10893 %}
10894
10895 // This pattern is automatically generated from aarch64_ad.m4
10896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10897 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10898 iRegL src1, iRegL src2,
10899 immI src3) %{
10900 match(Set dst (XorL src1 (RShiftL src2 src3)));
10901
10902 ins_cost(1.9 * INSN_COST);
10903 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10904
10905 ins_encode %{
10906 __ eor(as_Register($dst$$reg),
10907 as_Register($src1$$reg),
10908 as_Register($src2$$reg),
10909 Assembler::ASR,
10910 $src3$$constant & 0x3f);
10911 %}
10912
10913 ins_pipe(ialu_reg_reg_shift);
10914 %}
10915
10916 // This pattern is automatically generated from aarch64_ad.m4
10917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10918 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10919 iRegIorL2I src1, iRegIorL2I src2,
10920 immI src3) %{
10921 match(Set dst (XorI src1 (LShiftI src2 src3)));
10922
10923 ins_cost(1.9 * INSN_COST);
10924 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10925
10926 ins_encode %{
10927 __ eorw(as_Register($dst$$reg),
10928 as_Register($src1$$reg),
10929 as_Register($src2$$reg),
10930 Assembler::LSL,
10931 $src3$$constant & 0x1f);
10932 %}
10933
10934 ins_pipe(ialu_reg_reg_shift);
10935 %}
10936
10937 // This pattern is automatically generated from aarch64_ad.m4
10938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10939 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10940 iRegL src1, iRegL src2,
10941 immI src3) %{
10942 match(Set dst (XorL src1 (LShiftL src2 src3)));
10943
10944 ins_cost(1.9 * INSN_COST);
10945 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10946
10947 ins_encode %{
10948 __ eor(as_Register($dst$$reg),
10949 as_Register($src1$$reg),
10950 as_Register($src2$$reg),
10951 Assembler::LSL,
10952 $src3$$constant & 0x3f);
10953 %}
10954
10955 ins_pipe(ialu_reg_reg_shift);
10956 %}
10957
10958 // This pattern is automatically generated from aarch64_ad.m4
10959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10960 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10961 iRegIorL2I src1, iRegIorL2I src2,
10962 immI src3) %{
10963 match(Set dst (XorI src1 (RotateRight src2 src3)));
10964
10965 ins_cost(1.9 * INSN_COST);
10966 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10967
10968 ins_encode %{
10969 __ eorw(as_Register($dst$$reg),
10970 as_Register($src1$$reg),
10971 as_Register($src2$$reg),
10972 Assembler::ROR,
10973 $src3$$constant & 0x1f);
10974 %}
10975
10976 ins_pipe(ialu_reg_reg_shift);
10977 %}
10978
10979 // This pattern is automatically generated from aarch64_ad.m4
10980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10981 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10982 iRegL src1, iRegL src2,
10983 immI src3) %{
10984 match(Set dst (XorL src1 (RotateRight src2 src3)));
10985
10986 ins_cost(1.9 * INSN_COST);
10987 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10988
10989 ins_encode %{
10990 __ eor(as_Register($dst$$reg),
10991 as_Register($src1$$reg),
10992 as_Register($src2$$reg),
10993 Assembler::ROR,
10994 $src3$$constant & 0x3f);
10995 %}
10996
10997 ins_pipe(ialu_reg_reg_shift);
10998 %}
10999
11000 // This pattern is automatically generated from aarch64_ad.m4
11001 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11002 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11003 iRegIorL2I src1, iRegIorL2I src2,
11004 immI src3) %{
11005 match(Set dst (OrI src1 (URShiftI src2 src3)));
11006
11007 ins_cost(1.9 * INSN_COST);
11008 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11009
11010 ins_encode %{
11011 __ orrw(as_Register($dst$$reg),
11012 as_Register($src1$$reg),
11013 as_Register($src2$$reg),
11014 Assembler::LSR,
11015 $src3$$constant & 0x1f);
11016 %}
11017
11018 ins_pipe(ialu_reg_reg_shift);
11019 %}
11020
11021 // This pattern is automatically generated from aarch64_ad.m4
11022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11023 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11024 iRegL src1, iRegL src2,
11025 immI src3) %{
11026 match(Set dst (OrL src1 (URShiftL src2 src3)));
11027
11028 ins_cost(1.9 * INSN_COST);
11029 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11030
11031 ins_encode %{
11032 __ orr(as_Register($dst$$reg),
11033 as_Register($src1$$reg),
11034 as_Register($src2$$reg),
11035 Assembler::LSR,
11036 $src3$$constant & 0x3f);
11037 %}
11038
11039 ins_pipe(ialu_reg_reg_shift);
11040 %}
11041
11042 // This pattern is automatically generated from aarch64_ad.m4
11043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11044 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11045 iRegIorL2I src1, iRegIorL2I src2,
11046 immI src3) %{
11047 match(Set dst (OrI src1 (RShiftI src2 src3)));
11048
11049 ins_cost(1.9 * INSN_COST);
11050 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11051
11052 ins_encode %{
11053 __ orrw(as_Register($dst$$reg),
11054 as_Register($src1$$reg),
11055 as_Register($src2$$reg),
11056 Assembler::ASR,
11057 $src3$$constant & 0x1f);
11058 %}
11059
11060 ins_pipe(ialu_reg_reg_shift);
11061 %}
11062
11063 // This pattern is automatically generated from aarch64_ad.m4
11064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11065 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11066 iRegL src1, iRegL src2,
11067 immI src3) %{
11068 match(Set dst (OrL src1 (RShiftL src2 src3)));
11069
11070 ins_cost(1.9 * INSN_COST);
11071 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11072
11073 ins_encode %{
11074 __ orr(as_Register($dst$$reg),
11075 as_Register($src1$$reg),
11076 as_Register($src2$$reg),
11077 Assembler::ASR,
11078 $src3$$constant & 0x3f);
11079 %}
11080
11081 ins_pipe(ialu_reg_reg_shift);
11082 %}
11083
11084 // This pattern is automatically generated from aarch64_ad.m4
11085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11086 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11087 iRegIorL2I src1, iRegIorL2I src2,
11088 immI src3) %{
11089 match(Set dst (OrI src1 (LShiftI src2 src3)));
11090
11091 ins_cost(1.9 * INSN_COST);
11092 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11093
11094 ins_encode %{
11095 __ orrw(as_Register($dst$$reg),
11096 as_Register($src1$$reg),
11097 as_Register($src2$$reg),
11098 Assembler::LSL,
11099 $src3$$constant & 0x1f);
11100 %}
11101
11102 ins_pipe(ialu_reg_reg_shift);
11103 %}
11104
11105 // This pattern is automatically generated from aarch64_ad.m4
11106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11107 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11108 iRegL src1, iRegL src2,
11109 immI src3) %{
11110 match(Set dst (OrL src1 (LShiftL src2 src3)));
11111
11112 ins_cost(1.9 * INSN_COST);
11113 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11114
11115 ins_encode %{
11116 __ orr(as_Register($dst$$reg),
11117 as_Register($src1$$reg),
11118 as_Register($src2$$reg),
11119 Assembler::LSL,
11120 $src3$$constant & 0x3f);
11121 %}
11122
11123 ins_pipe(ialu_reg_reg_shift);
11124 %}
11125
11126 // This pattern is automatically generated from aarch64_ad.m4
11127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11128 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11129 iRegIorL2I src1, iRegIorL2I src2,
11130 immI src3) %{
11131 match(Set dst (OrI src1 (RotateRight src2 src3)));
11132
11133 ins_cost(1.9 * INSN_COST);
11134 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11135
11136 ins_encode %{
11137 __ orrw(as_Register($dst$$reg),
11138 as_Register($src1$$reg),
11139 as_Register($src2$$reg),
11140 Assembler::ROR,
11141 $src3$$constant & 0x1f);
11142 %}
11143
11144 ins_pipe(ialu_reg_reg_shift);
11145 %}
11146
11147 // This pattern is automatically generated from aarch64_ad.m4
11148 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11149 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11150 iRegL src1, iRegL src2,
11151 immI src3) %{
11152 match(Set dst (OrL src1 (RotateRight src2 src3)));
11153
11154 ins_cost(1.9 * INSN_COST);
11155 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11156
11157 ins_encode %{
11158 __ orr(as_Register($dst$$reg),
11159 as_Register($src1$$reg),
11160 as_Register($src2$$reg),
11161 Assembler::ROR,
11162 $src3$$constant & 0x3f);
11163 %}
11164
11165 ins_pipe(ialu_reg_reg_shift);
11166 %}
11167
11168 // This pattern is automatically generated from aarch64_ad.m4
11169 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11170 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11171 iRegIorL2I src1, iRegIorL2I src2,
11172 immI src3) %{
11173 match(Set dst (AddI src1 (URShiftI src2 src3)));
11174
11175 ins_cost(1.9 * INSN_COST);
11176 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11177
11178 ins_encode %{
11179 __ addw(as_Register($dst$$reg),
11180 as_Register($src1$$reg),
11181 as_Register($src2$$reg),
11182 Assembler::LSR,
11183 $src3$$constant & 0x1f);
11184 %}
11185
11186 ins_pipe(ialu_reg_reg_shift);
11187 %}
11188
11189 // This pattern is automatically generated from aarch64_ad.m4
11190 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11191 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11192 iRegL src1, iRegL src2,
11193 immI src3) %{
11194 match(Set dst (AddL src1 (URShiftL src2 src3)));
11195
11196 ins_cost(1.9 * INSN_COST);
11197 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11198
11199 ins_encode %{
11200 __ add(as_Register($dst$$reg),
11201 as_Register($src1$$reg),
11202 as_Register($src2$$reg),
11203 Assembler::LSR,
11204 $src3$$constant & 0x3f);
11205 %}
11206
11207 ins_pipe(ialu_reg_reg_shift);
11208 %}
11209
11210 // This pattern is automatically generated from aarch64_ad.m4
11211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11212 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11213 iRegIorL2I src1, iRegIorL2I src2,
11214 immI src3) %{
11215 match(Set dst (AddI src1 (RShiftI src2 src3)));
11216
11217 ins_cost(1.9 * INSN_COST);
11218 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11219
11220 ins_encode %{
11221 __ addw(as_Register($dst$$reg),
11222 as_Register($src1$$reg),
11223 as_Register($src2$$reg),
11224 Assembler::ASR,
11225 $src3$$constant & 0x1f);
11226 %}
11227
11228 ins_pipe(ialu_reg_reg_shift);
11229 %}
11230
11231 // This pattern is automatically generated from aarch64_ad.m4
11232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11233 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11234 iRegL src1, iRegL src2,
11235 immI src3) %{
11236 match(Set dst (AddL src1 (RShiftL src2 src3)));
11237
11238 ins_cost(1.9 * INSN_COST);
11239 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11240
11241 ins_encode %{
11242 __ add(as_Register($dst$$reg),
11243 as_Register($src1$$reg),
11244 as_Register($src2$$reg),
11245 Assembler::ASR,
11246 $src3$$constant & 0x3f);
11247 %}
11248
11249 ins_pipe(ialu_reg_reg_shift);
11250 %}
11251
11252 // This pattern is automatically generated from aarch64_ad.m4
11253 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11254 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11255 iRegIorL2I src1, iRegIorL2I src2,
11256 immI src3) %{
11257 match(Set dst (AddI src1 (LShiftI src2 src3)));
11258
11259 ins_cost(1.9 * INSN_COST);
11260 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11261
11262 ins_encode %{
11263 __ addw(as_Register($dst$$reg),
11264 as_Register($src1$$reg),
11265 as_Register($src2$$reg),
11266 Assembler::LSL,
11267 $src3$$constant & 0x1f);
11268 %}
11269
11270 ins_pipe(ialu_reg_reg_shift);
11271 %}
11272
11273 // This pattern is automatically generated from aarch64_ad.m4
11274 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11275 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11276 iRegL src1, iRegL src2,
11277 immI src3) %{
11278 match(Set dst (AddL src1 (LShiftL src2 src3)));
11279
11280 ins_cost(1.9 * INSN_COST);
11281 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11282
11283 ins_encode %{
11284 __ add(as_Register($dst$$reg),
11285 as_Register($src1$$reg),
11286 as_Register($src2$$reg),
11287 Assembler::LSL,
11288 $src3$$constant & 0x3f);
11289 %}
11290
11291 ins_pipe(ialu_reg_reg_shift);
11292 %}
11293
11294 // This pattern is automatically generated from aarch64_ad.m4
11295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11296 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11297 iRegIorL2I src1, iRegIorL2I src2,
11298 immI src3) %{
11299 match(Set dst (SubI src1 (URShiftI src2 src3)));
11300
11301 ins_cost(1.9 * INSN_COST);
11302 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11303
11304 ins_encode %{
11305 __ subw(as_Register($dst$$reg),
11306 as_Register($src1$$reg),
11307 as_Register($src2$$reg),
11308 Assembler::LSR,
11309 $src3$$constant & 0x1f);
11310 %}
11311
11312 ins_pipe(ialu_reg_reg_shift);
11313 %}
11314
11315 // This pattern is automatically generated from aarch64_ad.m4
11316 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11317 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11318 iRegL src1, iRegL src2,
11319 immI src3) %{
11320 match(Set dst (SubL src1 (URShiftL src2 src3)));
11321
11322 ins_cost(1.9 * INSN_COST);
11323 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11324
11325 ins_encode %{
11326 __ sub(as_Register($dst$$reg),
11327 as_Register($src1$$reg),
11328 as_Register($src2$$reg),
11329 Assembler::LSR,
11330 $src3$$constant & 0x3f);
11331 %}
11332
11333 ins_pipe(ialu_reg_reg_shift);
11334 %}
11335
11336 // This pattern is automatically generated from aarch64_ad.m4
11337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11338 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11339 iRegIorL2I src1, iRegIorL2I src2,
11340 immI src3) %{
11341 match(Set dst (SubI src1 (RShiftI src2 src3)));
11342
11343 ins_cost(1.9 * INSN_COST);
11344 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11345
11346 ins_encode %{
11347 __ subw(as_Register($dst$$reg),
11348 as_Register($src1$$reg),
11349 as_Register($src2$$reg),
11350 Assembler::ASR,
11351 $src3$$constant & 0x1f);
11352 %}
11353
11354 ins_pipe(ialu_reg_reg_shift);
11355 %}
11356
11357 // This pattern is automatically generated from aarch64_ad.m4
11358 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11359 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11360 iRegL src1, iRegL src2,
11361 immI src3) %{
11362 match(Set dst (SubL src1 (RShiftL src2 src3)));
11363
11364 ins_cost(1.9 * INSN_COST);
11365 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11366
11367 ins_encode %{
11368 __ sub(as_Register($dst$$reg),
11369 as_Register($src1$$reg),
11370 as_Register($src2$$reg),
11371 Assembler::ASR,
11372 $src3$$constant & 0x3f);
11373 %}
11374
11375 ins_pipe(ialu_reg_reg_shift);
11376 %}
11377
11378 // This pattern is automatically generated from aarch64_ad.m4
11379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11380 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11381 iRegIorL2I src1, iRegIorL2I src2,
11382 immI src3) %{
11383 match(Set dst (SubI src1 (LShiftI src2 src3)));
11384
11385 ins_cost(1.9 * INSN_COST);
11386 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11387
11388 ins_encode %{
11389 __ subw(as_Register($dst$$reg),
11390 as_Register($src1$$reg),
11391 as_Register($src2$$reg),
11392 Assembler::LSL,
11393 $src3$$constant & 0x1f);
11394 %}
11395
11396 ins_pipe(ialu_reg_reg_shift);
11397 %}
11398
11399 // This pattern is automatically generated from aarch64_ad.m4
11400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11401 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11402 iRegL src1, iRegL src2,
11403 immI src3) %{
11404 match(Set dst (SubL src1 (LShiftL src2 src3)));
11405
11406 ins_cost(1.9 * INSN_COST);
11407 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11408
11409 ins_encode %{
11410 __ sub(as_Register($dst$$reg),
11411 as_Register($src1$$reg),
11412 as_Register($src2$$reg),
11413 Assembler::LSL,
11414 $src3$$constant & 0x3f);
11415 %}
11416
11417 ins_pipe(ialu_reg_reg_shift);
11418 %}
11419
11420 // This pattern is automatically generated from aarch64_ad.m4
11421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11422
11423 // Shift Left followed by Shift Right.
11424 // This idiom is used by the compiler for the i2b bytecode etc.
11425 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11426 %{
11427 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11428 ins_cost(INSN_COST * 2);
11429 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11430 ins_encode %{
11431 int lshift = $lshift_count$$constant & 63;
11432 int rshift = $rshift_count$$constant & 63;
11433 int s = 63 - lshift;
11434 int r = (rshift - lshift) & 63;
11435 __ sbfm(as_Register($dst$$reg),
11436 as_Register($src$$reg),
11437 r, s);
11438 %}
11439
11440 ins_pipe(ialu_reg_shift);
11441 %}
11442
11443 // This pattern is automatically generated from aarch64_ad.m4
11444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11445
11446 // Shift Left followed by Shift Right.
11447 // This idiom is used by the compiler for the i2b bytecode etc.
11448 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11449 %{
11450 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11451 ins_cost(INSN_COST * 2);
11452 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11453 ins_encode %{
11454 int lshift = $lshift_count$$constant & 31;
11455 int rshift = $rshift_count$$constant & 31;
11456 int s = 31 - lshift;
11457 int r = (rshift - lshift) & 31;
11458 __ sbfmw(as_Register($dst$$reg),
11459 as_Register($src$$reg),
11460 r, s);
11461 %}
11462
11463 ins_pipe(ialu_reg_shift);
11464 %}
11465
11466 // This pattern is automatically generated from aarch64_ad.m4
11467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11468
11469 // Shift Left followed by Shift Right.
11470 // This idiom is used by the compiler for the i2b bytecode etc.
11471 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11472 %{
11473 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11474 ins_cost(INSN_COST * 2);
11475 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11476 ins_encode %{
11477 int lshift = $lshift_count$$constant & 63;
11478 int rshift = $rshift_count$$constant & 63;
11479 int s = 63 - lshift;
11480 int r = (rshift - lshift) & 63;
11481 __ ubfm(as_Register($dst$$reg),
11482 as_Register($src$$reg),
11483 r, s);
11484 %}
11485
11486 ins_pipe(ialu_reg_shift);
11487 %}
11488
11489 // This pattern is automatically generated from aarch64_ad.m4
11490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11491
11492 // Shift Left followed by Shift Right.
11493 // This idiom is used by the compiler for the i2b bytecode etc.
11494 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11495 %{
11496 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11497 ins_cost(INSN_COST * 2);
11498 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11499 ins_encode %{
11500 int lshift = $lshift_count$$constant & 31;
11501 int rshift = $rshift_count$$constant & 31;
11502 int s = 31 - lshift;
11503 int r = (rshift - lshift) & 31;
11504 __ ubfmw(as_Register($dst$$reg),
11505 as_Register($src$$reg),
11506 r, s);
11507 %}
11508
11509 ins_pipe(ialu_reg_shift);
11510 %}
11511
11512 // Bitfield extract with shift & mask
11513
11514 // This pattern is automatically generated from aarch64_ad.m4
11515 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11516 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11517 %{
11518 match(Set dst (AndI (URShiftI src rshift) mask));
11519 // Make sure we are not going to exceed what ubfxw can do.
11520 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11521
11522 ins_cost(INSN_COST);
11523 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11524 ins_encode %{
11525 int rshift = $rshift$$constant & 31;
11526 intptr_t mask = $mask$$constant;
11527 int width = exact_log2(mask+1);
11528 __ ubfxw(as_Register($dst$$reg),
11529 as_Register($src$$reg), rshift, width);
11530 %}
11531 ins_pipe(ialu_reg_shift);
11532 %}
11533
11534 // This pattern is automatically generated from aarch64_ad.m4
11535 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11536 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11537 %{
11538 match(Set dst (AndL (URShiftL src rshift) mask));
11539 // Make sure we are not going to exceed what ubfx can do.
11540 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11541
11542 ins_cost(INSN_COST);
11543 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11544 ins_encode %{
11545 int rshift = $rshift$$constant & 63;
11546 intptr_t mask = $mask$$constant;
11547 int width = exact_log2_long(mask+1);
11548 __ ubfx(as_Register($dst$$reg),
11549 as_Register($src$$reg), rshift, width);
11550 %}
11551 ins_pipe(ialu_reg_shift);
11552 %}
11553
11554
11555 // This pattern is automatically generated from aarch64_ad.m4
11556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11557
11558 // We can use ubfx when extending an And with a mask when we know mask
11559 // is positive. We know that because immI_bitmask guarantees it.
11560 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11561 %{
11562 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11563 // Make sure we are not going to exceed what ubfxw can do.
11564 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11565
11566 ins_cost(INSN_COST * 2);
11567 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11568 ins_encode %{
11569 int rshift = $rshift$$constant & 31;
11570 intptr_t mask = $mask$$constant;
11571 int width = exact_log2(mask+1);
11572 __ ubfx(as_Register($dst$$reg),
11573 as_Register($src$$reg), rshift, width);
11574 %}
11575 ins_pipe(ialu_reg_shift);
11576 %}
11577
11578
11579 // This pattern is automatically generated from aarch64_ad.m4
11580 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11581
11582 // We can use ubfiz when masking by a positive number and then left shifting the result.
11583 // We know that the mask is positive because immI_bitmask guarantees it.
11584 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11585 %{
11586 match(Set dst (LShiftI (AndI src mask) lshift));
11587 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11588
11589 ins_cost(INSN_COST);
11590 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11591 ins_encode %{
11592 int lshift = $lshift$$constant & 31;
11593 intptr_t mask = $mask$$constant;
11594 int width = exact_log2(mask+1);
11595 __ ubfizw(as_Register($dst$$reg),
11596 as_Register($src$$reg), lshift, width);
11597 %}
11598 ins_pipe(ialu_reg_shift);
11599 %}
11600
11601 // This pattern is automatically generated from aarch64_ad.m4
11602 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11603
11604 // We can use ubfiz when masking by a positive number and then left shifting the result.
11605 // We know that the mask is positive because immL_bitmask guarantees it.
11606 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11607 %{
11608 match(Set dst (LShiftL (AndL src mask) lshift));
11609 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11610
11611 ins_cost(INSN_COST);
11612 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11613 ins_encode %{
11614 int lshift = $lshift$$constant & 63;
11615 intptr_t mask = $mask$$constant;
11616 int width = exact_log2_long(mask+1);
11617 __ ubfiz(as_Register($dst$$reg),
11618 as_Register($src$$reg), lshift, width);
11619 %}
11620 ins_pipe(ialu_reg_shift);
11621 %}
11622
11623 // This pattern is automatically generated from aarch64_ad.m4
11624 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11625
11626 // We can use ubfiz when masking by a positive number and then left shifting the result.
11627 // We know that the mask is positive because immI_bitmask guarantees it.
11628 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11629 %{
11630 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11631 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11632
11633 ins_cost(INSN_COST);
11634 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11635 ins_encode %{
11636 int lshift = $lshift$$constant & 31;
11637 intptr_t mask = $mask$$constant;
11638 int width = exact_log2(mask+1);
11639 __ ubfizw(as_Register($dst$$reg),
11640 as_Register($src$$reg), lshift, width);
11641 %}
11642 ins_pipe(ialu_reg_shift);
11643 %}
11644
11645 // This pattern is automatically generated from aarch64_ad.m4
11646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11647
11648 // We can use ubfiz when masking by a positive number and then left shifting the result.
11649 // We know that the mask is positive because immL_bitmask guarantees it.
11650 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11651 %{
11652 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11653 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11654
11655 ins_cost(INSN_COST);
11656 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11657 ins_encode %{
11658 int lshift = $lshift$$constant & 63;
11659 intptr_t mask = $mask$$constant;
11660 int width = exact_log2_long(mask+1);
11661 __ ubfiz(as_Register($dst$$reg),
11662 as_Register($src$$reg), lshift, width);
11663 %}
11664 ins_pipe(ialu_reg_shift);
11665 %}
11666
11667
11668 // This pattern is automatically generated from aarch64_ad.m4
11669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11670
11671 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11672 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11673 %{
11674 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11675 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11676
11677 ins_cost(INSN_COST);
11678 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11679 ins_encode %{
11680 int lshift = $lshift$$constant & 63;
11681 intptr_t mask = $mask$$constant;
11682 int width = exact_log2(mask+1);
11683 __ ubfiz(as_Register($dst$$reg),
11684 as_Register($src$$reg), lshift, width);
11685 %}
11686 ins_pipe(ialu_reg_shift);
11687 %}
11688
11689 // This pattern is automatically generated from aarch64_ad.m4
11690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11691
11692 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11693 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11694 %{
11695 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11696 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11697
11698 ins_cost(INSN_COST);
11699 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11700 ins_encode %{
11701 int lshift = $lshift$$constant & 31;
11702 intptr_t mask = $mask$$constant;
11703 int width = exact_log2(mask+1);
11704 __ ubfiz(as_Register($dst$$reg),
11705 as_Register($src$$reg), lshift, width);
11706 %}
11707 ins_pipe(ialu_reg_shift);
11708 %}
11709
11710 // This pattern is automatically generated from aarch64_ad.m4
11711 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11712
11713 // Can skip int2long conversions after AND with small bitmask
11714 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11715 %{
11716 match(Set dst (ConvI2L (AndI src msk)));
11717 ins_cost(INSN_COST);
11718 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11719 ins_encode %{
11720 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11721 %}
11722 ins_pipe(ialu_reg_shift);
11723 %}
11724
11725
11726 // Rotations
11727
11728 // This pattern is automatically generated from aarch64_ad.m4
11729 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11730 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11731 %{
11732 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11733 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11734
11735 ins_cost(INSN_COST);
11736 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11737
11738 ins_encode %{
11739 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11740 $rshift$$constant & 63);
11741 %}
11742 ins_pipe(ialu_reg_reg_extr);
11743 %}
11744
11745
11746 // This pattern is automatically generated from aarch64_ad.m4
11747 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11748 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11749 %{
11750 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11751 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11752
11753 ins_cost(INSN_COST);
11754 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11755
11756 ins_encode %{
11757 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11758 $rshift$$constant & 31);
11759 %}
11760 ins_pipe(ialu_reg_reg_extr);
11761 %}
11762
11763
11764 // This pattern is automatically generated from aarch64_ad.m4
11765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11766 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11767 %{
11768 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11769 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11770
11771 ins_cost(INSN_COST);
11772 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11773
11774 ins_encode %{
11775 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11776 $rshift$$constant & 63);
11777 %}
11778 ins_pipe(ialu_reg_reg_extr);
11779 %}
11780
11781
11782 // This pattern is automatically generated from aarch64_ad.m4
11783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11784 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11785 %{
11786 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11787 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11788
11789 ins_cost(INSN_COST);
11790 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11791
11792 ins_encode %{
11793 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11794 $rshift$$constant & 31);
11795 %}
11796 ins_pipe(ialu_reg_reg_extr);
11797 %}
11798
11799 // This pattern is automatically generated from aarch64_ad.m4
11800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11801 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11802 %{
11803 match(Set dst (RotateRight src shift));
11804
11805 ins_cost(INSN_COST);
11806 format %{ "ror $dst, $src, $shift" %}
11807
11808 ins_encode %{
11809 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11810 $shift$$constant & 0x1f);
11811 %}
11812 ins_pipe(ialu_reg_reg_vshift);
11813 %}
11814
11815 // This pattern is automatically generated from aarch64_ad.m4
11816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11817 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11818 %{
11819 match(Set dst (RotateRight src shift));
11820
11821 ins_cost(INSN_COST);
11822 format %{ "ror $dst, $src, $shift" %}
11823
11824 ins_encode %{
11825 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11826 $shift$$constant & 0x3f);
11827 %}
11828 ins_pipe(ialu_reg_reg_vshift);
11829 %}
11830
11831 // This pattern is automatically generated from aarch64_ad.m4
11832 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11833 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11834 %{
11835 match(Set dst (RotateRight src shift));
11836
11837 ins_cost(INSN_COST);
11838 format %{ "ror $dst, $src, $shift" %}
11839
11840 ins_encode %{
11841 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11842 %}
11843 ins_pipe(ialu_reg_reg_vshift);
11844 %}
11845
11846 // This pattern is automatically generated from aarch64_ad.m4
11847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11848 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11849 %{
11850 match(Set dst (RotateRight src shift));
11851
11852 ins_cost(INSN_COST);
11853 format %{ "ror $dst, $src, $shift" %}
11854
11855 ins_encode %{
11856 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11857 %}
11858 ins_pipe(ialu_reg_reg_vshift);
11859 %}
11860
11861 // This pattern is automatically generated from aarch64_ad.m4
11862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11863 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11864 %{
11865 match(Set dst (RotateLeft src shift));
11866
11867 ins_cost(INSN_COST);
11868 format %{ "rol $dst, $src, $shift" %}
11869
11870 ins_encode %{
11871 __ subw(rscratch1, zr, as_Register($shift$$reg));
11872 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11873 %}
11874 ins_pipe(ialu_reg_reg_vshift);
11875 %}
11876
11877 // This pattern is automatically generated from aarch64_ad.m4
11878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11879 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11880 %{
11881 match(Set dst (RotateLeft src shift));
11882
11883 ins_cost(INSN_COST);
11884 format %{ "rol $dst, $src, $shift" %}
11885
11886 ins_encode %{
11887 __ subw(rscratch1, zr, as_Register($shift$$reg));
11888 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11889 %}
11890 ins_pipe(ialu_reg_reg_vshift);
11891 %}
11892
11893
11894 // Add/subtract (extended)
11895
11896 // This pattern is automatically generated from aarch64_ad.m4
11897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11898 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11899 %{
11900 match(Set dst (AddL src1 (ConvI2L src2)));
11901 ins_cost(INSN_COST);
11902 format %{ "add $dst, $src1, $src2, sxtw" %}
11903
11904 ins_encode %{
11905 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11906 as_Register($src2$$reg), ext::sxtw);
11907 %}
11908 ins_pipe(ialu_reg_reg);
11909 %}
11910
11911 // This pattern is automatically generated from aarch64_ad.m4
11912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11913 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11914 %{
11915 match(Set dst (SubL src1 (ConvI2L src2)));
11916 ins_cost(INSN_COST);
11917 format %{ "sub $dst, $src1, $src2, sxtw" %}
11918
11919 ins_encode %{
11920 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11921 as_Register($src2$$reg), ext::sxtw);
11922 %}
11923 ins_pipe(ialu_reg_reg);
11924 %}
11925
11926 // This pattern is automatically generated from aarch64_ad.m4
11927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11928 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11929 %{
11930 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11931 ins_cost(INSN_COST);
11932 format %{ "add $dst, $src1, $src2, sxth" %}
11933
11934 ins_encode %{
11935 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11936 as_Register($src2$$reg), ext::sxth);
11937 %}
11938 ins_pipe(ialu_reg_reg);
11939 %}
11940
11941 // This pattern is automatically generated from aarch64_ad.m4
11942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11943 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11944 %{
11945 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11946 ins_cost(INSN_COST);
11947 format %{ "add $dst, $src1, $src2, sxtb" %}
11948
11949 ins_encode %{
11950 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11951 as_Register($src2$$reg), ext::sxtb);
11952 %}
11953 ins_pipe(ialu_reg_reg);
11954 %}
11955
11956 // This pattern is automatically generated from aarch64_ad.m4
11957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11958 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11959 %{
11960 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11961 ins_cost(INSN_COST);
11962 format %{ "add $dst, $src1, $src2, uxtb" %}
11963
11964 ins_encode %{
11965 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11966 as_Register($src2$$reg), ext::uxtb);
11967 %}
11968 ins_pipe(ialu_reg_reg);
11969 %}
11970
11971 // This pattern is automatically generated from aarch64_ad.m4
11972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11973 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11974 %{
11975 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11976 ins_cost(INSN_COST);
11977 format %{ "add $dst, $src1, $src2, sxth" %}
11978
11979 ins_encode %{
11980 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11981 as_Register($src2$$reg), ext::sxth);
11982 %}
11983 ins_pipe(ialu_reg_reg);
11984 %}
11985
11986 // This pattern is automatically generated from aarch64_ad.m4
11987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11988 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11989 %{
11990 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11991 ins_cost(INSN_COST);
11992 format %{ "add $dst, $src1, $src2, sxtw" %}
11993
11994 ins_encode %{
11995 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11996 as_Register($src2$$reg), ext::sxtw);
11997 %}
11998 ins_pipe(ialu_reg_reg);
11999 %}
12000
12001 // This pattern is automatically generated from aarch64_ad.m4
12002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12003 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12004 %{
12005 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12006 ins_cost(INSN_COST);
12007 format %{ "add $dst, $src1, $src2, sxtb" %}
12008
12009 ins_encode %{
12010 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12011 as_Register($src2$$reg), ext::sxtb);
12012 %}
12013 ins_pipe(ialu_reg_reg);
12014 %}
12015
12016 // This pattern is automatically generated from aarch64_ad.m4
12017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12018 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12019 %{
12020 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12021 ins_cost(INSN_COST);
12022 format %{ "add $dst, $src1, $src2, uxtb" %}
12023
12024 ins_encode %{
12025 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12026 as_Register($src2$$reg), ext::uxtb);
12027 %}
12028 ins_pipe(ialu_reg_reg);
12029 %}
12030
12031 // This pattern is automatically generated from aarch64_ad.m4
12032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12033 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12034 %{
12035 match(Set dst (AddI src1 (AndI src2 mask)));
12036 ins_cost(INSN_COST);
12037 format %{ "addw $dst, $src1, $src2, uxtb" %}
12038
12039 ins_encode %{
12040 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12041 as_Register($src2$$reg), ext::uxtb);
12042 %}
12043 ins_pipe(ialu_reg_reg);
12044 %}
12045
12046 // This pattern is automatically generated from aarch64_ad.m4
12047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12048 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12049 %{
12050 match(Set dst (AddI src1 (AndI src2 mask)));
12051 ins_cost(INSN_COST);
12052 format %{ "addw $dst, $src1, $src2, uxth" %}
12053
12054 ins_encode %{
12055 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12056 as_Register($src2$$reg), ext::uxth);
12057 %}
12058 ins_pipe(ialu_reg_reg);
12059 %}
12060
12061 // This pattern is automatically generated from aarch64_ad.m4
12062 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12063 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12064 %{
12065 match(Set dst (AddL src1 (AndL src2 mask)));
12066 ins_cost(INSN_COST);
12067 format %{ "add $dst, $src1, $src2, uxtb" %}
12068
12069 ins_encode %{
12070 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12071 as_Register($src2$$reg), ext::uxtb);
12072 %}
12073 ins_pipe(ialu_reg_reg);
12074 %}
12075
12076 // This pattern is automatically generated from aarch64_ad.m4
12077 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12078 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12079 %{
12080 match(Set dst (AddL src1 (AndL src2 mask)));
12081 ins_cost(INSN_COST);
12082 format %{ "add $dst, $src1, $src2, uxth" %}
12083
12084 ins_encode %{
12085 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12086 as_Register($src2$$reg), ext::uxth);
12087 %}
12088 ins_pipe(ialu_reg_reg);
12089 %}
12090
12091 // This pattern is automatically generated from aarch64_ad.m4
12092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12093 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12094 %{
12095 match(Set dst (AddL src1 (AndL src2 mask)));
12096 ins_cost(INSN_COST);
12097 format %{ "add $dst, $src1, $src2, uxtw" %}
12098
12099 ins_encode %{
12100 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12101 as_Register($src2$$reg), ext::uxtw);
12102 %}
12103 ins_pipe(ialu_reg_reg);
12104 %}
12105
12106 // This pattern is automatically generated from aarch64_ad.m4
12107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12108 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12109 %{
12110 match(Set dst (SubI src1 (AndI src2 mask)));
12111 ins_cost(INSN_COST);
12112 format %{ "subw $dst, $src1, $src2, uxtb" %}
12113
12114 ins_encode %{
12115 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12116 as_Register($src2$$reg), ext::uxtb);
12117 %}
12118 ins_pipe(ialu_reg_reg);
12119 %}
12120
12121 // This pattern is automatically generated from aarch64_ad.m4
12122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12123 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12124 %{
12125 match(Set dst (SubI src1 (AndI src2 mask)));
12126 ins_cost(INSN_COST);
12127 format %{ "subw $dst, $src1, $src2, uxth" %}
12128
12129 ins_encode %{
12130 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12131 as_Register($src2$$reg), ext::uxth);
12132 %}
12133 ins_pipe(ialu_reg_reg);
12134 %}
12135
12136 // This pattern is automatically generated from aarch64_ad.m4
12137 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12138 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12139 %{
12140 match(Set dst (SubL src1 (AndL src2 mask)));
12141 ins_cost(INSN_COST);
12142 format %{ "sub $dst, $src1, $src2, uxtb" %}
12143
12144 ins_encode %{
12145 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12146 as_Register($src2$$reg), ext::uxtb);
12147 %}
12148 ins_pipe(ialu_reg_reg);
12149 %}
12150
12151 // This pattern is automatically generated from aarch64_ad.m4
12152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12153 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12154 %{
12155 match(Set dst (SubL src1 (AndL src2 mask)));
12156 ins_cost(INSN_COST);
12157 format %{ "sub $dst, $src1, $src2, uxth" %}
12158
12159 ins_encode %{
12160 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12161 as_Register($src2$$reg), ext::uxth);
12162 %}
12163 ins_pipe(ialu_reg_reg);
12164 %}
12165
12166 // This pattern is automatically generated from aarch64_ad.m4
12167 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12168 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12169 %{
12170 match(Set dst (SubL src1 (AndL src2 mask)));
12171 ins_cost(INSN_COST);
12172 format %{ "sub $dst, $src1, $src2, uxtw" %}
12173
12174 ins_encode %{
12175 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12176 as_Register($src2$$reg), ext::uxtw);
12177 %}
12178 ins_pipe(ialu_reg_reg);
12179 %}
12180
12181
12182 // This pattern is automatically generated from aarch64_ad.m4
12183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12184 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12185 %{
12186 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12187 ins_cost(1.9 * INSN_COST);
12188 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12189
12190 ins_encode %{
12191 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12192 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12193 %}
12194 ins_pipe(ialu_reg_reg_shift);
12195 %}
12196
12197 // This pattern is automatically generated from aarch64_ad.m4
12198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12199 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12200 %{
12201 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12202 ins_cost(1.9 * INSN_COST);
12203 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12204
12205 ins_encode %{
12206 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12207 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12208 %}
12209 ins_pipe(ialu_reg_reg_shift);
12210 %}
12211
12212 // This pattern is automatically generated from aarch64_ad.m4
12213 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12214 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12215 %{
12216 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12217 ins_cost(1.9 * INSN_COST);
12218 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12219
12220 ins_encode %{
12221 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12222 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12223 %}
12224 ins_pipe(ialu_reg_reg_shift);
12225 %}
12226
12227 // This pattern is automatically generated from aarch64_ad.m4
12228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12229 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12230 %{
12231 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12232 ins_cost(1.9 * INSN_COST);
12233 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12234
12235 ins_encode %{
12236 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12237 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12238 %}
12239 ins_pipe(ialu_reg_reg_shift);
12240 %}
12241
12242 // This pattern is automatically generated from aarch64_ad.m4
12243 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12244 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12245 %{
12246 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12247 ins_cost(1.9 * INSN_COST);
12248 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12249
12250 ins_encode %{
12251 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12252 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12253 %}
12254 ins_pipe(ialu_reg_reg_shift);
12255 %}
12256
12257 // This pattern is automatically generated from aarch64_ad.m4
12258 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12259 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12260 %{
12261 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12262 ins_cost(1.9 * INSN_COST);
12263 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12264
12265 ins_encode %{
12266 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12267 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12268 %}
12269 ins_pipe(ialu_reg_reg_shift);
12270 %}
12271
12272 // This pattern is automatically generated from aarch64_ad.m4
12273 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12274 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12275 %{
12276 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12277 ins_cost(1.9 * INSN_COST);
12278 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12279
12280 ins_encode %{
12281 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12282 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12283 %}
12284 ins_pipe(ialu_reg_reg_shift);
12285 %}
12286
12287 // This pattern is automatically generated from aarch64_ad.m4
12288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12289 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12290 %{
12291 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12292 ins_cost(1.9 * INSN_COST);
12293 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12294
12295 ins_encode %{
12296 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12297 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12298 %}
12299 ins_pipe(ialu_reg_reg_shift);
12300 %}
12301
12302 // This pattern is automatically generated from aarch64_ad.m4
12303 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12304 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12305 %{
12306 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12307 ins_cost(1.9 * INSN_COST);
12308 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12309
12310 ins_encode %{
12311 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12312 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12313 %}
12314 ins_pipe(ialu_reg_reg_shift);
12315 %}
12316
12317 // This pattern is automatically generated from aarch64_ad.m4
12318 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12319 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12320 %{
12321 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12322 ins_cost(1.9 * INSN_COST);
12323 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12324
12325 ins_encode %{
12326 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12327 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12328 %}
12329 ins_pipe(ialu_reg_reg_shift);
12330 %}
12331
12332 // This pattern is automatically generated from aarch64_ad.m4
12333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12334 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12335 %{
12336 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12337 ins_cost(1.9 * INSN_COST);
12338 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12339
12340 ins_encode %{
12341 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12342 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12343 %}
12344 ins_pipe(ialu_reg_reg_shift);
12345 %}
12346
12347 // This pattern is automatically generated from aarch64_ad.m4
12348 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12349 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12350 %{
12351 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12352 ins_cost(1.9 * INSN_COST);
12353 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12354
12355 ins_encode %{
12356 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12357 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12358 %}
12359 ins_pipe(ialu_reg_reg_shift);
12360 %}
12361
12362 // This pattern is automatically generated from aarch64_ad.m4
12363 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12364 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12365 %{
12366 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12367 ins_cost(1.9 * INSN_COST);
12368 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12369
12370 ins_encode %{
12371 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12372 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12373 %}
12374 ins_pipe(ialu_reg_reg_shift);
12375 %}
12376
12377 // This pattern is automatically generated from aarch64_ad.m4
12378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12379 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12380 %{
12381 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12382 ins_cost(1.9 * INSN_COST);
12383 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12384
12385 ins_encode %{
12386 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12387 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12388 %}
12389 ins_pipe(ialu_reg_reg_shift);
12390 %}
12391
12392 // This pattern is automatically generated from aarch64_ad.m4
12393 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12394 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12395 %{
12396 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12397 ins_cost(1.9 * INSN_COST);
12398 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12399
12400 ins_encode %{
12401 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12402 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12403 %}
12404 ins_pipe(ialu_reg_reg_shift);
12405 %}
12406
12407 // This pattern is automatically generated from aarch64_ad.m4
12408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12409 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12410 %{
12411 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12412 ins_cost(1.9 * INSN_COST);
12413 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12414
12415 ins_encode %{
12416 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12417 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12418 %}
12419 ins_pipe(ialu_reg_reg_shift);
12420 %}
12421
12422 // This pattern is automatically generated from aarch64_ad.m4
12423 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12424 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12425 %{
12426 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12427 ins_cost(1.9 * INSN_COST);
12428 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12429
12430 ins_encode %{
12431 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12432 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12433 %}
12434 ins_pipe(ialu_reg_reg_shift);
12435 %}
12436
12437 // This pattern is automatically generated from aarch64_ad.m4
12438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12439 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12440 %{
12441 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12442 ins_cost(1.9 * INSN_COST);
12443 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12444
12445 ins_encode %{
12446 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12447 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12448 %}
12449 ins_pipe(ialu_reg_reg_shift);
12450 %}
12451
12452 // This pattern is automatically generated from aarch64_ad.m4
12453 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12454 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12455 %{
12456 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12457 ins_cost(1.9 * INSN_COST);
12458 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12459
12460 ins_encode %{
12461 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12462 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12463 %}
12464 ins_pipe(ialu_reg_reg_shift);
12465 %}
12466
12467 // This pattern is automatically generated from aarch64_ad.m4
12468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12469 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12470 %{
12471 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12472 ins_cost(1.9 * INSN_COST);
12473 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12474
12475 ins_encode %{
12476 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12477 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12478 %}
12479 ins_pipe(ialu_reg_reg_shift);
12480 %}
12481
12482 // This pattern is automatically generated from aarch64_ad.m4
12483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12484 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12485 %{
12486 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12487 ins_cost(1.9 * INSN_COST);
12488 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12489
12490 ins_encode %{
12491 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12492 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12493 %}
12494 ins_pipe(ialu_reg_reg_shift);
12495 %}
12496
12497 // This pattern is automatically generated from aarch64_ad.m4
12498 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12499 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12500 %{
12501 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12502 ins_cost(1.9 * INSN_COST);
12503 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12504
12505 ins_encode %{
12506 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12507 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12508 %}
12509 ins_pipe(ialu_reg_reg_shift);
12510 %}
12511
12512 // This pattern is automatically generated from aarch64_ad.m4
12513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12514 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12515 %{
12516 effect(DEF dst, USE src1, USE src2, USE cr);
12517 ins_cost(INSN_COST * 2);
12518 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12519
12520 ins_encode %{
12521 __ cselw($dst$$Register,
12522 $src1$$Register,
12523 $src2$$Register,
12524 Assembler::LT);
12525 %}
12526 ins_pipe(icond_reg_reg);
12527 %}
12528
12529 // This pattern is automatically generated from aarch64_ad.m4
12530 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12531 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12532 %{
12533 effect(DEF dst, USE src1, USE src2, USE cr);
12534 ins_cost(INSN_COST * 2);
12535 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12536
12537 ins_encode %{
12538 __ cselw($dst$$Register,
12539 $src1$$Register,
12540 $src2$$Register,
12541 Assembler::GT);
12542 %}
12543 ins_pipe(icond_reg_reg);
12544 %}
12545
12546 // This pattern is automatically generated from aarch64_ad.m4
12547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12548 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12549 %{
12550 effect(DEF dst, USE src1, USE cr);
12551 ins_cost(INSN_COST * 2);
12552 format %{ "cselw $dst, $src1, zr lt\t" %}
12553
12554 ins_encode %{
12555 __ cselw($dst$$Register,
12556 $src1$$Register,
12557 zr,
12558 Assembler::LT);
12559 %}
12560 ins_pipe(icond_reg);
12561 %}
12562
12563 // This pattern is automatically generated from aarch64_ad.m4
12564 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12565 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12566 %{
12567 effect(DEF dst, USE src1, USE cr);
12568 ins_cost(INSN_COST * 2);
12569 format %{ "cselw $dst, $src1, zr gt\t" %}
12570
12571 ins_encode %{
12572 __ cselw($dst$$Register,
12573 $src1$$Register,
12574 zr,
12575 Assembler::GT);
12576 %}
12577 ins_pipe(icond_reg);
12578 %}
12579
12580 // This pattern is automatically generated from aarch64_ad.m4
12581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12582 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12583 %{
12584 effect(DEF dst, USE src1, USE cr);
12585 ins_cost(INSN_COST * 2);
12586 format %{ "csincw $dst, $src1, zr le\t" %}
12587
12588 ins_encode %{
12589 __ csincw($dst$$Register,
12590 $src1$$Register,
12591 zr,
12592 Assembler::LE);
12593 %}
12594 ins_pipe(icond_reg);
12595 %}
12596
12597 // This pattern is automatically generated from aarch64_ad.m4
12598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12599 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12600 %{
12601 effect(DEF dst, USE src1, USE cr);
12602 ins_cost(INSN_COST * 2);
12603 format %{ "csincw $dst, $src1, zr gt\t" %}
12604
12605 ins_encode %{
12606 __ csincw($dst$$Register,
12607 $src1$$Register,
12608 zr,
12609 Assembler::GT);
12610 %}
12611 ins_pipe(icond_reg);
12612 %}
12613
12614 // This pattern is automatically generated from aarch64_ad.m4
12615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12616 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12617 %{
12618 effect(DEF dst, USE src1, USE cr);
12619 ins_cost(INSN_COST * 2);
12620 format %{ "csinvw $dst, $src1, zr lt\t" %}
12621
12622 ins_encode %{
12623 __ csinvw($dst$$Register,
12624 $src1$$Register,
12625 zr,
12626 Assembler::LT);
12627 %}
12628 ins_pipe(icond_reg);
12629 %}
12630
12631 // This pattern is automatically generated from aarch64_ad.m4
12632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12633 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12634 %{
12635 effect(DEF dst, USE src1, USE cr);
12636 ins_cost(INSN_COST * 2);
12637 format %{ "csinvw $dst, $src1, zr ge\t" %}
12638
12639 ins_encode %{
12640 __ csinvw($dst$$Register,
12641 $src1$$Register,
12642 zr,
12643 Assembler::GE);
12644 %}
12645 ins_pipe(icond_reg);
12646 %}
12647
12648 // This pattern is automatically generated from aarch64_ad.m4
12649 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12650 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12651 %{
12652 match(Set dst (MinI src imm));
12653 ins_cost(INSN_COST * 3);
12654 expand %{
12655 rFlagsReg cr;
12656 compI_reg_imm0(cr, src);
12657 cmovI_reg_imm0_lt(dst, src, cr);
12658 %}
12659 %}
12660
12661 // This pattern is automatically generated from aarch64_ad.m4
12662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12663 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12664 %{
12665 match(Set dst (MinI imm src));
12666 ins_cost(INSN_COST * 3);
12667 expand %{
12668 rFlagsReg cr;
12669 compI_reg_imm0(cr, src);
12670 cmovI_reg_imm0_lt(dst, src, cr);
12671 %}
12672 %}
12673
12674 // This pattern is automatically generated from aarch64_ad.m4
12675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12676 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12677 %{
12678 match(Set dst (MinI src imm));
12679 ins_cost(INSN_COST * 3);
12680 expand %{
12681 rFlagsReg cr;
12682 compI_reg_imm0(cr, src);
12683 cmovI_reg_imm1_le(dst, src, cr);
12684 %}
12685 %}
12686
12687 // This pattern is automatically generated from aarch64_ad.m4
12688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12689 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12690 %{
12691 match(Set dst (MinI imm src));
12692 ins_cost(INSN_COST * 3);
12693 expand %{
12694 rFlagsReg cr;
12695 compI_reg_imm0(cr, src);
12696 cmovI_reg_imm1_le(dst, src, cr);
12697 %}
12698 %}
12699
12700 // This pattern is automatically generated from aarch64_ad.m4
12701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12702 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12703 %{
12704 match(Set dst (MinI src imm));
12705 ins_cost(INSN_COST * 3);
12706 expand %{
12707 rFlagsReg cr;
12708 compI_reg_imm0(cr, src);
12709 cmovI_reg_immM1_lt(dst, src, cr);
12710 %}
12711 %}
12712
12713 // This pattern is automatically generated from aarch64_ad.m4
12714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12715 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12716 %{
12717 match(Set dst (MinI imm src));
12718 ins_cost(INSN_COST * 3);
12719 expand %{
12720 rFlagsReg cr;
12721 compI_reg_imm0(cr, src);
12722 cmovI_reg_immM1_lt(dst, src, cr);
12723 %}
12724 %}
12725
12726 // This pattern is automatically generated from aarch64_ad.m4
12727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12728 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12729 %{
12730 match(Set dst (MaxI src imm));
12731 ins_cost(INSN_COST * 3);
12732 expand %{
12733 rFlagsReg cr;
12734 compI_reg_imm0(cr, src);
12735 cmovI_reg_imm0_gt(dst, src, cr);
12736 %}
12737 %}
12738
12739 // This pattern is automatically generated from aarch64_ad.m4
12740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12741 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12742 %{
12743 match(Set dst (MaxI imm src));
12744 ins_cost(INSN_COST * 3);
12745 expand %{
12746 rFlagsReg cr;
12747 compI_reg_imm0(cr, src);
12748 cmovI_reg_imm0_gt(dst, src, cr);
12749 %}
12750 %}
12751
12752 // This pattern is automatically generated from aarch64_ad.m4
12753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12754 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12755 %{
12756 match(Set dst (MaxI src imm));
12757 ins_cost(INSN_COST * 3);
12758 expand %{
12759 rFlagsReg cr;
12760 compI_reg_imm0(cr, src);
12761 cmovI_reg_imm1_gt(dst, src, cr);
12762 %}
12763 %}
12764
12765 // This pattern is automatically generated from aarch64_ad.m4
12766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12767 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12768 %{
12769 match(Set dst (MaxI imm src));
12770 ins_cost(INSN_COST * 3);
12771 expand %{
12772 rFlagsReg cr;
12773 compI_reg_imm0(cr, src);
12774 cmovI_reg_imm1_gt(dst, src, cr);
12775 %}
12776 %}
12777
12778 // This pattern is automatically generated from aarch64_ad.m4
12779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12780 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12781 %{
12782 match(Set dst (MaxI src imm));
12783 ins_cost(INSN_COST * 3);
12784 expand %{
12785 rFlagsReg cr;
12786 compI_reg_imm0(cr, src);
12787 cmovI_reg_immM1_ge(dst, src, cr);
12788 %}
12789 %}
12790
12791 // This pattern is automatically generated from aarch64_ad.m4
12792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12793 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12794 %{
12795 match(Set dst (MaxI imm src));
12796 ins_cost(INSN_COST * 3);
12797 expand %{
12798 rFlagsReg cr;
12799 compI_reg_imm0(cr, src);
12800 cmovI_reg_immM1_ge(dst, src, cr);
12801 %}
12802 %}
12803
12804 // This pattern is automatically generated from aarch64_ad.m4
12805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12806 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12807 %{
12808 match(Set dst (ReverseI src));
12809 ins_cost(INSN_COST);
12810 format %{ "rbitw $dst, $src" %}
12811 ins_encode %{
12812 __ rbitw($dst$$Register, $src$$Register);
12813 %}
12814 ins_pipe(ialu_reg);
12815 %}
12816
12817 // This pattern is automatically generated from aarch64_ad.m4
12818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12819 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12820 %{
12821 match(Set dst (ReverseL src));
12822 ins_cost(INSN_COST);
12823 format %{ "rbit $dst, $src" %}
12824 ins_encode %{
12825 __ rbit($dst$$Register, $src$$Register);
12826 %}
12827 ins_pipe(ialu_reg);
12828 %}
12829
12830
12831 // END This section of the file is automatically generated. Do not edit --------------
12832
12833
12834 // ============================================================================
12835 // Floating Point Arithmetic Instructions
12836
12837 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12838 match(Set dst (AddHF src1 src2));
12839 format %{ "faddh $dst, $src1, $src2" %}
12840 ins_encode %{
12841 __ faddh($dst$$FloatRegister,
12842 $src1$$FloatRegister,
12843 $src2$$FloatRegister);
12844 %}
12845 ins_pipe(fp_dop_reg_reg_s);
12846 %}
12847
12848 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12849 match(Set dst (AddF src1 src2));
12850
12851 ins_cost(INSN_COST * 5);
12852 format %{ "fadds $dst, $src1, $src2" %}
12853
12854 ins_encode %{
12855 __ fadds(as_FloatRegister($dst$$reg),
12856 as_FloatRegister($src1$$reg),
12857 as_FloatRegister($src2$$reg));
12858 %}
12859
12860 ins_pipe(fp_dop_reg_reg_s);
12861 %}
12862
12863 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12864 match(Set dst (AddD src1 src2));
12865
12866 ins_cost(INSN_COST * 5);
12867 format %{ "faddd $dst, $src1, $src2" %}
12868
12869 ins_encode %{
12870 __ faddd(as_FloatRegister($dst$$reg),
12871 as_FloatRegister($src1$$reg),
12872 as_FloatRegister($src2$$reg));
12873 %}
12874
12875 ins_pipe(fp_dop_reg_reg_d);
12876 %}
12877
12878 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12879 match(Set dst (SubHF src1 src2));
12880 format %{ "fsubh $dst, $src1, $src2" %}
12881 ins_encode %{
12882 __ fsubh($dst$$FloatRegister,
12883 $src1$$FloatRegister,
12884 $src2$$FloatRegister);
12885 %}
12886 ins_pipe(fp_dop_reg_reg_s);
12887 %}
12888
12889 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12890 match(Set dst (SubF src1 src2));
12891
12892 ins_cost(INSN_COST * 5);
12893 format %{ "fsubs $dst, $src1, $src2" %}
12894
12895 ins_encode %{
12896 __ fsubs(as_FloatRegister($dst$$reg),
12897 as_FloatRegister($src1$$reg),
12898 as_FloatRegister($src2$$reg));
12899 %}
12900
12901 ins_pipe(fp_dop_reg_reg_s);
12902 %}
12903
12904 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12905 match(Set dst (SubD src1 src2));
12906
12907 ins_cost(INSN_COST * 5);
12908 format %{ "fsubd $dst, $src1, $src2" %}
12909
12910 ins_encode %{
12911 __ fsubd(as_FloatRegister($dst$$reg),
12912 as_FloatRegister($src1$$reg),
12913 as_FloatRegister($src2$$reg));
12914 %}
12915
12916 ins_pipe(fp_dop_reg_reg_d);
12917 %}
12918
12919 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12920 match(Set dst (MulHF src1 src2));
12921 format %{ "fmulh $dst, $src1, $src2" %}
12922 ins_encode %{
12923 __ fmulh($dst$$FloatRegister,
12924 $src1$$FloatRegister,
12925 $src2$$FloatRegister);
12926 %}
12927 ins_pipe(fp_dop_reg_reg_s);
12928 %}
12929
12930 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12931 match(Set dst (MulF src1 src2));
12932
12933 ins_cost(INSN_COST * 6);
12934 format %{ "fmuls $dst, $src1, $src2" %}
12935
12936 ins_encode %{
12937 __ fmuls(as_FloatRegister($dst$$reg),
12938 as_FloatRegister($src1$$reg),
12939 as_FloatRegister($src2$$reg));
12940 %}
12941
12942 ins_pipe(fp_dop_reg_reg_s);
12943 %}
12944
12945 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12946 match(Set dst (MulD src1 src2));
12947
12948 ins_cost(INSN_COST * 6);
12949 format %{ "fmuld $dst, $src1, $src2" %}
12950
12951 ins_encode %{
12952 __ fmuld(as_FloatRegister($dst$$reg),
12953 as_FloatRegister($src1$$reg),
12954 as_FloatRegister($src2$$reg));
12955 %}
12956
12957 ins_pipe(fp_dop_reg_reg_d);
12958 %}
12959
12960 // src1 * src2 + src3 (half-precision float)
12961 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12962 match(Set dst (FmaHF src3 (Binary src1 src2)));
12963 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12964 ins_encode %{
12965 assert(UseFMA, "Needs FMA instructions support.");
12966 __ fmaddh($dst$$FloatRegister,
12967 $src1$$FloatRegister,
12968 $src2$$FloatRegister,
12969 $src3$$FloatRegister);
12970 %}
12971 ins_pipe(pipe_class_default);
12972 %}
12973
12974 // src1 * src2 + src3
12975 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12976 match(Set dst (FmaF src3 (Binary src1 src2)));
12977
12978 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12979
12980 ins_encode %{
12981 assert(UseFMA, "Needs FMA instructions support.");
12982 __ fmadds(as_FloatRegister($dst$$reg),
12983 as_FloatRegister($src1$$reg),
12984 as_FloatRegister($src2$$reg),
12985 as_FloatRegister($src3$$reg));
12986 %}
12987
12988 ins_pipe(pipe_class_default);
12989 %}
12990
12991 // src1 * src2 + src3
12992 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12993 match(Set dst (FmaD src3 (Binary src1 src2)));
12994
12995 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12996
12997 ins_encode %{
12998 assert(UseFMA, "Needs FMA instructions support.");
12999 __ fmaddd(as_FloatRegister($dst$$reg),
13000 as_FloatRegister($src1$$reg),
13001 as_FloatRegister($src2$$reg),
13002 as_FloatRegister($src3$$reg));
13003 %}
13004
13005 ins_pipe(pipe_class_default);
13006 %}
13007
13008 // src1 * (-src2) + src3
13009 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13010 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13011 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13012
13013 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13014
13015 ins_encode %{
13016 assert(UseFMA, "Needs FMA instructions support.");
13017 __ fmsubs(as_FloatRegister($dst$$reg),
13018 as_FloatRegister($src1$$reg),
13019 as_FloatRegister($src2$$reg),
13020 as_FloatRegister($src3$$reg));
13021 %}
13022
13023 ins_pipe(pipe_class_default);
13024 %}
13025
13026 // src1 * (-src2) + src3
13027 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13028 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13029 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13030
13031 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13032
13033 ins_encode %{
13034 assert(UseFMA, "Needs FMA instructions support.");
13035 __ fmsubd(as_FloatRegister($dst$$reg),
13036 as_FloatRegister($src1$$reg),
13037 as_FloatRegister($src2$$reg),
13038 as_FloatRegister($src3$$reg));
13039 %}
13040
13041 ins_pipe(pipe_class_default);
13042 %}
13043
13044 // src1 * (-src2) - src3
13045 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13046 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13047 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13048
13049 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13050
13051 ins_encode %{
13052 assert(UseFMA, "Needs FMA instructions support.");
13053 __ fnmadds(as_FloatRegister($dst$$reg),
13054 as_FloatRegister($src1$$reg),
13055 as_FloatRegister($src2$$reg),
13056 as_FloatRegister($src3$$reg));
13057 %}
13058
13059 ins_pipe(pipe_class_default);
13060 %}
13061
13062 // src1 * (-src2) - src3
13063 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13064 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13065 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13066
13067 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13068
13069 ins_encode %{
13070 assert(UseFMA, "Needs FMA instructions support.");
13071 __ fnmaddd(as_FloatRegister($dst$$reg),
13072 as_FloatRegister($src1$$reg),
13073 as_FloatRegister($src2$$reg),
13074 as_FloatRegister($src3$$reg));
13075 %}
13076
13077 ins_pipe(pipe_class_default);
13078 %}
13079
13080 // src1 * src2 - src3
13081 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13082 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13083
13084 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13085
13086 ins_encode %{
13087 assert(UseFMA, "Needs FMA instructions support.");
13088 __ fnmsubs(as_FloatRegister($dst$$reg),
13089 as_FloatRegister($src1$$reg),
13090 as_FloatRegister($src2$$reg),
13091 as_FloatRegister($src3$$reg));
13092 %}
13093
13094 ins_pipe(pipe_class_default);
13095 %}
13096
13097 // src1 * src2 - src3
13098 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13099 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13100
13101 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13102
13103 ins_encode %{
13104 assert(UseFMA, "Needs FMA instructions support.");
13105 // n.b. insn name should be fnmsubd
13106 __ fnmsub(as_FloatRegister($dst$$reg),
13107 as_FloatRegister($src1$$reg),
13108 as_FloatRegister($src2$$reg),
13109 as_FloatRegister($src3$$reg));
13110 %}
13111
13112 ins_pipe(pipe_class_default);
13113 %}
13114
13115 // Math.max(HH)H (half-precision float)
13116 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13117 match(Set dst (MaxHF src1 src2));
13118 format %{ "fmaxh $dst, $src1, $src2" %}
13119 ins_encode %{
13120 __ fmaxh($dst$$FloatRegister,
13121 $src1$$FloatRegister,
13122 $src2$$FloatRegister);
13123 %}
13124 ins_pipe(fp_dop_reg_reg_s);
13125 %}
13126
13127 // Math.min(HH)H (half-precision float)
13128 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13129 match(Set dst (MinHF src1 src2));
13130 format %{ "fminh $dst, $src1, $src2" %}
13131 ins_encode %{
13132 __ fminh($dst$$FloatRegister,
13133 $src1$$FloatRegister,
13134 $src2$$FloatRegister);
13135 %}
13136 ins_pipe(fp_dop_reg_reg_s);
13137 %}
13138
13139 // Math.max(FF)F
13140 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13141 match(Set dst (MaxF src1 src2));
13142
13143 format %{ "fmaxs $dst, $src1, $src2" %}
13144 ins_encode %{
13145 __ fmaxs(as_FloatRegister($dst$$reg),
13146 as_FloatRegister($src1$$reg),
13147 as_FloatRegister($src2$$reg));
13148 %}
13149
13150 ins_pipe(fp_dop_reg_reg_s);
13151 %}
13152
13153 // Math.min(FF)F
13154 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13155 match(Set dst (MinF src1 src2));
13156
13157 format %{ "fmins $dst, $src1, $src2" %}
13158 ins_encode %{
13159 __ fmins(as_FloatRegister($dst$$reg),
13160 as_FloatRegister($src1$$reg),
13161 as_FloatRegister($src2$$reg));
13162 %}
13163
13164 ins_pipe(fp_dop_reg_reg_s);
13165 %}
13166
13167 // Math.max(DD)D
13168 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13169 match(Set dst (MaxD src1 src2));
13170
13171 format %{ "fmaxd $dst, $src1, $src2" %}
13172 ins_encode %{
13173 __ fmaxd(as_FloatRegister($dst$$reg),
13174 as_FloatRegister($src1$$reg),
13175 as_FloatRegister($src2$$reg));
13176 %}
13177
13178 ins_pipe(fp_dop_reg_reg_d);
13179 %}
13180
13181 // Math.min(DD)D
13182 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13183 match(Set dst (MinD src1 src2));
13184
13185 format %{ "fmind $dst, $src1, $src2" %}
13186 ins_encode %{
13187 __ fmind(as_FloatRegister($dst$$reg),
13188 as_FloatRegister($src1$$reg),
13189 as_FloatRegister($src2$$reg));
13190 %}
13191
13192 ins_pipe(fp_dop_reg_reg_d);
13193 %}
13194
13195 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13196 match(Set dst (DivHF src1 src2));
13197 format %{ "fdivh $dst, $src1, $src2" %}
13198 ins_encode %{
13199 __ fdivh($dst$$FloatRegister,
13200 $src1$$FloatRegister,
13201 $src2$$FloatRegister);
13202 %}
13203 ins_pipe(fp_div_s);
13204 %}
13205
13206 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13207 match(Set dst (DivF src1 src2));
13208
13209 ins_cost(INSN_COST * 18);
13210 format %{ "fdivs $dst, $src1, $src2" %}
13211
13212 ins_encode %{
13213 __ fdivs(as_FloatRegister($dst$$reg),
13214 as_FloatRegister($src1$$reg),
13215 as_FloatRegister($src2$$reg));
13216 %}
13217
13218 ins_pipe(fp_div_s);
13219 %}
13220
13221 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13222 match(Set dst (DivD src1 src2));
13223
13224 ins_cost(INSN_COST * 32);
13225 format %{ "fdivd $dst, $src1, $src2" %}
13226
13227 ins_encode %{
13228 __ fdivd(as_FloatRegister($dst$$reg),
13229 as_FloatRegister($src1$$reg),
13230 as_FloatRegister($src2$$reg));
13231 %}
13232
13233 ins_pipe(fp_div_d);
13234 %}
13235
13236 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13237 match(Set dst (NegF src));
13238
13239 ins_cost(INSN_COST * 3);
13240 format %{ "fneg $dst, $src" %}
13241
13242 ins_encode %{
13243 __ fnegs(as_FloatRegister($dst$$reg),
13244 as_FloatRegister($src$$reg));
13245 %}
13246
13247 ins_pipe(fp_uop_s);
13248 %}
13249
13250 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13251 match(Set dst (NegD src));
13252
13253 ins_cost(INSN_COST * 3);
13254 format %{ "fnegd $dst, $src" %}
13255
13256 ins_encode %{
13257 __ fnegd(as_FloatRegister($dst$$reg),
13258 as_FloatRegister($src$$reg));
13259 %}
13260
13261 ins_pipe(fp_uop_d);
13262 %}
13263
13264 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13265 %{
13266 match(Set dst (AbsI src));
13267
13268 effect(KILL cr);
13269 ins_cost(INSN_COST * 2);
13270 format %{ "cmpw $src, zr\n\t"
13271 "cnegw $dst, $src, Assembler::LT\t# int abs"
13272 %}
13273
13274 ins_encode %{
13275 __ cmpw(as_Register($src$$reg), zr);
13276 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13277 %}
13278 ins_pipe(pipe_class_default);
13279 %}
13280
13281 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13282 %{
13283 match(Set dst (AbsL src));
13284
13285 effect(KILL cr);
13286 ins_cost(INSN_COST * 2);
13287 format %{ "cmp $src, zr\n\t"
13288 "cneg $dst, $src, Assembler::LT\t# long abs"
13289 %}
13290
13291 ins_encode %{
13292 __ cmp(as_Register($src$$reg), zr);
13293 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13294 %}
13295 ins_pipe(pipe_class_default);
13296 %}
13297
13298 instruct absF_reg(vRegF dst, vRegF src) %{
13299 match(Set dst (AbsF src));
13300
13301 ins_cost(INSN_COST * 3);
13302 format %{ "fabss $dst, $src" %}
13303 ins_encode %{
13304 __ fabss(as_FloatRegister($dst$$reg),
13305 as_FloatRegister($src$$reg));
13306 %}
13307
13308 ins_pipe(fp_uop_s);
13309 %}
13310
13311 instruct absD_reg(vRegD dst, vRegD src) %{
13312 match(Set dst (AbsD src));
13313
13314 ins_cost(INSN_COST * 3);
13315 format %{ "fabsd $dst, $src" %}
13316 ins_encode %{
13317 __ fabsd(as_FloatRegister($dst$$reg),
13318 as_FloatRegister($src$$reg));
13319 %}
13320
13321 ins_pipe(fp_uop_d);
13322 %}
13323
13324 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13325 match(Set dst (AbsF (SubF src1 src2)));
13326
13327 ins_cost(INSN_COST * 3);
13328 format %{ "fabds $dst, $src1, $src2" %}
13329 ins_encode %{
13330 __ fabds(as_FloatRegister($dst$$reg),
13331 as_FloatRegister($src1$$reg),
13332 as_FloatRegister($src2$$reg));
13333 %}
13334
13335 ins_pipe(fp_uop_s);
13336 %}
13337
13338 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13339 match(Set dst (AbsD (SubD src1 src2)));
13340
13341 ins_cost(INSN_COST * 3);
13342 format %{ "fabdd $dst, $src1, $src2" %}
13343 ins_encode %{
13344 __ fabdd(as_FloatRegister($dst$$reg),
13345 as_FloatRegister($src1$$reg),
13346 as_FloatRegister($src2$$reg));
13347 %}
13348
13349 ins_pipe(fp_uop_d);
13350 %}
13351
13352 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13353 match(Set dst (SqrtD src));
13354
13355 ins_cost(INSN_COST * 50);
13356 format %{ "fsqrtd $dst, $src" %}
13357 ins_encode %{
13358 __ fsqrtd(as_FloatRegister($dst$$reg),
13359 as_FloatRegister($src$$reg));
13360 %}
13361
13362 ins_pipe(fp_div_s);
13363 %}
13364
13365 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13366 match(Set dst (SqrtF src));
13367
13368 ins_cost(INSN_COST * 50);
13369 format %{ "fsqrts $dst, $src" %}
13370 ins_encode %{
13371 __ fsqrts(as_FloatRegister($dst$$reg),
13372 as_FloatRegister($src$$reg));
13373 %}
13374
13375 ins_pipe(fp_div_d);
13376 %}
13377
13378 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13379 match(Set dst (SqrtHF src));
13380 format %{ "fsqrth $dst, $src" %}
13381 ins_encode %{
13382 __ fsqrth($dst$$FloatRegister,
13383 $src$$FloatRegister);
13384 %}
13385 ins_pipe(fp_div_s);
13386 %}
13387
13388 // Math.rint, floor, ceil
13389 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13390 match(Set dst (RoundDoubleMode src rmode));
13391 format %{ "frint $dst, $src, $rmode" %}
13392 ins_encode %{
13393 switch ($rmode$$constant) {
13394 case RoundDoubleModeNode::rmode_rint:
13395 __ frintnd(as_FloatRegister($dst$$reg),
13396 as_FloatRegister($src$$reg));
13397 break;
13398 case RoundDoubleModeNode::rmode_floor:
13399 __ frintmd(as_FloatRegister($dst$$reg),
13400 as_FloatRegister($src$$reg));
13401 break;
13402 case RoundDoubleModeNode::rmode_ceil:
13403 __ frintpd(as_FloatRegister($dst$$reg),
13404 as_FloatRegister($src$$reg));
13405 break;
13406 }
13407 %}
13408 ins_pipe(fp_uop_d);
13409 %}
13410
13411 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13412 match(Set dst (CopySignD src1 (Binary src2 zero)));
13413 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13414 format %{ "CopySignD $dst $src1 $src2" %}
13415 ins_encode %{
13416 FloatRegister dst = as_FloatRegister($dst$$reg),
13417 src1 = as_FloatRegister($src1$$reg),
13418 src2 = as_FloatRegister($src2$$reg),
13419 zero = as_FloatRegister($zero$$reg);
13420 __ fnegd(dst, zero);
13421 __ bsl(dst, __ T8B, src2, src1);
13422 %}
13423 ins_pipe(fp_uop_d);
13424 %}
13425
13426 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13427 match(Set dst (CopySignF src1 src2));
13428 effect(TEMP_DEF dst, USE src1, USE src2);
13429 format %{ "CopySignF $dst $src1 $src2" %}
13430 ins_encode %{
13431 FloatRegister dst = as_FloatRegister($dst$$reg),
13432 src1 = as_FloatRegister($src1$$reg),
13433 src2 = as_FloatRegister($src2$$reg);
13434 __ movi(dst, __ T2S, 0x80, 24);
13435 __ bsl(dst, __ T8B, src2, src1);
13436 %}
13437 ins_pipe(fp_uop_d);
13438 %}
13439
13440 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13441 match(Set dst (SignumD src (Binary zero one)));
13442 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13443 format %{ "signumD $dst, $src" %}
13444 ins_encode %{
13445 FloatRegister src = as_FloatRegister($src$$reg),
13446 dst = as_FloatRegister($dst$$reg),
13447 zero = as_FloatRegister($zero$$reg),
13448 one = as_FloatRegister($one$$reg);
13449 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13450 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13451 // Bit selection instruction gets bit from "one" for each enabled bit in
13452 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13453 // NaN the whole "src" will be copied because "dst" is zero. For all other
13454 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13455 // from "src", and all other bits are copied from 1.0.
13456 __ bsl(dst, __ T8B, one, src);
13457 %}
13458 ins_pipe(fp_uop_d);
13459 %}
13460
13461 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13462 match(Set dst (SignumF src (Binary zero one)));
13463 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13464 format %{ "signumF $dst, $src" %}
13465 ins_encode %{
13466 FloatRegister src = as_FloatRegister($src$$reg),
13467 dst = as_FloatRegister($dst$$reg),
13468 zero = as_FloatRegister($zero$$reg),
13469 one = as_FloatRegister($one$$reg);
13470 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13471 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13472 // Bit selection instruction gets bit from "one" for each enabled bit in
13473 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13474 // NaN the whole "src" will be copied because "dst" is zero. For all other
13475 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13476 // from "src", and all other bits are copied from 1.0.
13477 __ bsl(dst, __ T8B, one, src);
13478 %}
13479 ins_pipe(fp_uop_d);
13480 %}
13481
13482 instruct onspinwait() %{
13483 match(OnSpinWait);
13484 ins_cost(INSN_COST);
13485
13486 format %{ "onspinwait" %}
13487
13488 ins_encode %{
13489 __ spin_wait();
13490 %}
13491 ins_pipe(pipe_class_empty);
13492 %}
13493
13494 // ============================================================================
13495 // Logical Instructions
13496
13497 // Integer Logical Instructions
13498
13499 // And Instructions
13500
13501
13502 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13503 match(Set dst (AndI src1 src2));
13504
13505 format %{ "andw $dst, $src1, $src2\t# int" %}
13506
13507 ins_cost(INSN_COST);
13508 ins_encode %{
13509 __ andw(as_Register($dst$$reg),
13510 as_Register($src1$$reg),
13511 as_Register($src2$$reg));
13512 %}
13513
13514 ins_pipe(ialu_reg_reg);
13515 %}
13516
13517 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13518 match(Set dst (AndI src1 src2));
13519
13520 format %{ "andsw $dst, $src1, $src2\t# int" %}
13521
13522 ins_cost(INSN_COST);
13523 ins_encode %{
13524 __ andw(as_Register($dst$$reg),
13525 as_Register($src1$$reg),
13526 (uint64_t)($src2$$constant));
13527 %}
13528
13529 ins_pipe(ialu_reg_imm);
13530 %}
13531
13532 // Or Instructions
13533
13534 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13535 match(Set dst (OrI src1 src2));
13536
13537 format %{ "orrw $dst, $src1, $src2\t# int" %}
13538
13539 ins_cost(INSN_COST);
13540 ins_encode %{
13541 __ orrw(as_Register($dst$$reg),
13542 as_Register($src1$$reg),
13543 as_Register($src2$$reg));
13544 %}
13545
13546 ins_pipe(ialu_reg_reg);
13547 %}
13548
13549 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13550 match(Set dst (OrI src1 src2));
13551
13552 format %{ "orrw $dst, $src1, $src2\t# int" %}
13553
13554 ins_cost(INSN_COST);
13555 ins_encode %{
13556 __ orrw(as_Register($dst$$reg),
13557 as_Register($src1$$reg),
13558 (uint64_t)($src2$$constant));
13559 %}
13560
13561 ins_pipe(ialu_reg_imm);
13562 %}
13563
13564 // Xor Instructions
13565
13566 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13567 match(Set dst (XorI src1 src2));
13568
13569 format %{ "eorw $dst, $src1, $src2\t# int" %}
13570
13571 ins_cost(INSN_COST);
13572 ins_encode %{
13573 __ eorw(as_Register($dst$$reg),
13574 as_Register($src1$$reg),
13575 as_Register($src2$$reg));
13576 %}
13577
13578 ins_pipe(ialu_reg_reg);
13579 %}
13580
13581 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13582 match(Set dst (XorI src1 src2));
13583
13584 format %{ "eorw $dst, $src1, $src2\t# int" %}
13585
13586 ins_cost(INSN_COST);
13587 ins_encode %{
13588 __ eorw(as_Register($dst$$reg),
13589 as_Register($src1$$reg),
13590 (uint64_t)($src2$$constant));
13591 %}
13592
13593 ins_pipe(ialu_reg_imm);
13594 %}
13595
13596 // Long Logical Instructions
13597 // TODO
13598
13599 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13600 match(Set dst (AndL src1 src2));
13601
13602 format %{ "and $dst, $src1, $src2\t# int" %}
13603
13604 ins_cost(INSN_COST);
13605 ins_encode %{
13606 __ andr(as_Register($dst$$reg),
13607 as_Register($src1$$reg),
13608 as_Register($src2$$reg));
13609 %}
13610
13611 ins_pipe(ialu_reg_reg);
13612 %}
13613
13614 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13615 match(Set dst (AndL src1 src2));
13616
13617 format %{ "and $dst, $src1, $src2\t# int" %}
13618
13619 ins_cost(INSN_COST);
13620 ins_encode %{
13621 __ andr(as_Register($dst$$reg),
13622 as_Register($src1$$reg),
13623 (uint64_t)($src2$$constant));
13624 %}
13625
13626 ins_pipe(ialu_reg_imm);
13627 %}
13628
13629 // Or Instructions
13630
13631 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13632 match(Set dst (OrL src1 src2));
13633
13634 format %{ "orr $dst, $src1, $src2\t# int" %}
13635
13636 ins_cost(INSN_COST);
13637 ins_encode %{
13638 __ orr(as_Register($dst$$reg),
13639 as_Register($src1$$reg),
13640 as_Register($src2$$reg));
13641 %}
13642
13643 ins_pipe(ialu_reg_reg);
13644 %}
13645
13646 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13647 match(Set dst (OrL src1 src2));
13648
13649 format %{ "orr $dst, $src1, $src2\t# int" %}
13650
13651 ins_cost(INSN_COST);
13652 ins_encode %{
13653 __ orr(as_Register($dst$$reg),
13654 as_Register($src1$$reg),
13655 (uint64_t)($src2$$constant));
13656 %}
13657
13658 ins_pipe(ialu_reg_imm);
13659 %}
13660
13661 // Xor Instructions
13662
13663 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13664 match(Set dst (XorL src1 src2));
13665
13666 format %{ "eor $dst, $src1, $src2\t# int" %}
13667
13668 ins_cost(INSN_COST);
13669 ins_encode %{
13670 __ eor(as_Register($dst$$reg),
13671 as_Register($src1$$reg),
13672 as_Register($src2$$reg));
13673 %}
13674
13675 ins_pipe(ialu_reg_reg);
13676 %}
13677
13678 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13679 match(Set dst (XorL src1 src2));
13680
13681 ins_cost(INSN_COST);
13682 format %{ "eor $dst, $src1, $src2\t# int" %}
13683
13684 ins_encode %{
13685 __ eor(as_Register($dst$$reg),
13686 as_Register($src1$$reg),
13687 (uint64_t)($src2$$constant));
13688 %}
13689
13690 ins_pipe(ialu_reg_imm);
13691 %}
13692
13693 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13694 %{
13695 match(Set dst (ConvI2L src));
13696
13697 ins_cost(INSN_COST);
13698 format %{ "sxtw $dst, $src\t# i2l" %}
13699 ins_encode %{
13700 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13701 %}
13702 ins_pipe(ialu_reg_shift);
13703 %}
13704
13705 // this pattern occurs in bigmath arithmetic
13706 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13707 %{
13708 match(Set dst (AndL (ConvI2L src) mask));
13709
13710 ins_cost(INSN_COST);
13711 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13712 ins_encode %{
13713 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13714 %}
13715
13716 ins_pipe(ialu_reg_shift);
13717 %}
13718
13719 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13720 match(Set dst (ConvL2I src));
13721
13722 ins_cost(INSN_COST);
13723 format %{ "movw $dst, $src \t// l2i" %}
13724
13725 ins_encode %{
13726 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13727 %}
13728
13729 ins_pipe(ialu_reg);
13730 %}
13731
13732 instruct convD2F_reg(vRegF dst, vRegD src) %{
13733 match(Set dst (ConvD2F src));
13734
13735 ins_cost(INSN_COST * 5);
13736 format %{ "fcvtd $dst, $src \t// d2f" %}
13737
13738 ins_encode %{
13739 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13740 %}
13741
13742 ins_pipe(fp_d2f);
13743 %}
13744
13745 instruct convF2D_reg(vRegD dst, vRegF src) %{
13746 match(Set dst (ConvF2D src));
13747
13748 ins_cost(INSN_COST * 5);
13749 format %{ "fcvts $dst, $src \t// f2d" %}
13750
13751 ins_encode %{
13752 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13753 %}
13754
13755 ins_pipe(fp_f2d);
13756 %}
13757
13758 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13759 match(Set dst (ConvF2I src));
13760
13761 ins_cost(INSN_COST * 5);
13762 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13763
13764 ins_encode %{
13765 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13766 %}
13767
13768 ins_pipe(fp_f2i);
13769 %}
13770
13771 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13772 match(Set dst (ConvF2L src));
13773
13774 ins_cost(INSN_COST * 5);
13775 format %{ "fcvtzs $dst, $src \t// f2l" %}
13776
13777 ins_encode %{
13778 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13779 %}
13780
13781 ins_pipe(fp_f2l);
13782 %}
13783
13784 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13785 match(Set dst (ConvF2HF src));
13786 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13787 "smov $dst, $tmp\t# move result from $tmp to $dst"
13788 %}
13789 effect(TEMP tmp);
13790 ins_encode %{
13791 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13792 %}
13793 ins_pipe(pipe_slow);
13794 %}
13795
13796 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13797 match(Set dst (ConvHF2F src));
13798 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13799 "fcvt $dst, $tmp\t# convert half to single precision"
13800 %}
13801 effect(TEMP tmp);
13802 ins_encode %{
13803 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13804 %}
13805 ins_pipe(pipe_slow);
13806 %}
13807
13808 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13809 match(Set dst (ConvI2F src));
13810
13811 ins_cost(INSN_COST * 5);
13812 format %{ "scvtfws $dst, $src \t// i2f" %}
13813
13814 ins_encode %{
13815 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13816 %}
13817
13818 ins_pipe(fp_i2f);
13819 %}
13820
13821 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13822 match(Set dst (ConvL2F src));
13823
13824 ins_cost(INSN_COST * 5);
13825 format %{ "scvtfs $dst, $src \t// l2f" %}
13826
13827 ins_encode %{
13828 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13829 %}
13830
13831 ins_pipe(fp_l2f);
13832 %}
13833
13834 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13835 match(Set dst (ConvD2I src));
13836
13837 ins_cost(INSN_COST * 5);
13838 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13839
13840 ins_encode %{
13841 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13842 %}
13843
13844 ins_pipe(fp_d2i);
13845 %}
13846
13847 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13848 match(Set dst (ConvD2L src));
13849
13850 ins_cost(INSN_COST * 5);
13851 format %{ "fcvtzd $dst, $src \t// d2l" %}
13852
13853 ins_encode %{
13854 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13855 %}
13856
13857 ins_pipe(fp_d2l);
13858 %}
13859
13860 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13861 match(Set dst (ConvI2D src));
13862
13863 ins_cost(INSN_COST * 5);
13864 format %{ "scvtfwd $dst, $src \t// i2d" %}
13865
13866 ins_encode %{
13867 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13868 %}
13869
13870 ins_pipe(fp_i2d);
13871 %}
13872
13873 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13874 match(Set dst (ConvL2D src));
13875
13876 ins_cost(INSN_COST * 5);
13877 format %{ "scvtfd $dst, $src \t// l2d" %}
13878
13879 ins_encode %{
13880 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13881 %}
13882
13883 ins_pipe(fp_l2d);
13884 %}
13885
13886 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13887 %{
13888 match(Set dst (RoundD src));
13889 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13890 format %{ "java_round_double $dst,$src"%}
13891 ins_encode %{
13892 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13893 as_FloatRegister($ftmp$$reg));
13894 %}
13895 ins_pipe(pipe_slow);
13896 %}
13897
13898 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13899 %{
13900 match(Set dst (RoundF src));
13901 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13902 format %{ "java_round_float $dst,$src"%}
13903 ins_encode %{
13904 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13905 as_FloatRegister($ftmp$$reg));
13906 %}
13907 ins_pipe(pipe_slow);
13908 %}
13909
13910 // stack <-> reg and reg <-> reg shuffles with no conversion
13911
13912 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13913
13914 match(Set dst (MoveF2I src));
13915
13916 effect(DEF dst, USE src);
13917
13918 ins_cost(4 * INSN_COST);
13919
13920 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13921
13922 ins_encode %{
13923 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13924 %}
13925
13926 ins_pipe(iload_reg_reg);
13927
13928 %}
13929
13930 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13931
13932 match(Set dst (MoveI2F src));
13933
13934 effect(DEF dst, USE src);
13935
13936 ins_cost(4 * INSN_COST);
13937
13938 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13939
13940 ins_encode %{
13941 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13942 %}
13943
13944 ins_pipe(pipe_class_memory);
13945
13946 %}
13947
13948 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13949
13950 match(Set dst (MoveD2L src));
13951
13952 effect(DEF dst, USE src);
13953
13954 ins_cost(4 * INSN_COST);
13955
13956 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13957
13958 ins_encode %{
13959 __ ldr($dst$$Register, Address(sp, $src$$disp));
13960 %}
13961
13962 ins_pipe(iload_reg_reg);
13963
13964 %}
13965
13966 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13967
13968 match(Set dst (MoveL2D src));
13969
13970 effect(DEF dst, USE src);
13971
13972 ins_cost(4 * INSN_COST);
13973
13974 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13975
13976 ins_encode %{
13977 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13978 %}
13979
13980 ins_pipe(pipe_class_memory);
13981
13982 %}
13983
13984 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13985
13986 match(Set dst (MoveF2I src));
13987
13988 effect(DEF dst, USE src);
13989
13990 ins_cost(INSN_COST);
13991
13992 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13993
13994 ins_encode %{
13995 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13996 %}
13997
13998 ins_pipe(pipe_class_memory);
13999
14000 %}
14001
14002 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14003
14004 match(Set dst (MoveI2F src));
14005
14006 effect(DEF dst, USE src);
14007
14008 ins_cost(INSN_COST);
14009
14010 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14011
14012 ins_encode %{
14013 __ strw($src$$Register, Address(sp, $dst$$disp));
14014 %}
14015
14016 ins_pipe(istore_reg_reg);
14017
14018 %}
14019
14020 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14021
14022 match(Set dst (MoveD2L src));
14023
14024 effect(DEF dst, USE src);
14025
14026 ins_cost(INSN_COST);
14027
14028 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14029
14030 ins_encode %{
14031 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14032 %}
14033
14034 ins_pipe(pipe_class_memory);
14035
14036 %}
14037
14038 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14039
14040 match(Set dst (MoveL2D src));
14041
14042 effect(DEF dst, USE src);
14043
14044 ins_cost(INSN_COST);
14045
14046 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14047
14048 ins_encode %{
14049 __ str($src$$Register, Address(sp, $dst$$disp));
14050 %}
14051
14052 ins_pipe(istore_reg_reg);
14053
14054 %}
14055
14056 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14057
14058 match(Set dst (MoveF2I src));
14059
14060 effect(DEF dst, USE src);
14061
14062 ins_cost(INSN_COST);
14063
14064 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14065
14066 ins_encode %{
14067 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14068 %}
14069
14070 ins_pipe(fp_f2i);
14071
14072 %}
14073
14074 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14075
14076 match(Set dst (MoveI2F src));
14077
14078 effect(DEF dst, USE src);
14079
14080 ins_cost(INSN_COST);
14081
14082 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14083
14084 ins_encode %{
14085 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14086 %}
14087
14088 ins_pipe(fp_i2f);
14089
14090 %}
14091
14092 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14093
14094 match(Set dst (MoveD2L src));
14095
14096 effect(DEF dst, USE src);
14097
14098 ins_cost(INSN_COST);
14099
14100 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14101
14102 ins_encode %{
14103 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14104 %}
14105
14106 ins_pipe(fp_d2l);
14107
14108 %}
14109
14110 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14111
14112 match(Set dst (MoveL2D src));
14113
14114 effect(DEF dst, USE src);
14115
14116 ins_cost(INSN_COST);
14117
14118 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14119
14120 ins_encode %{
14121 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14122 %}
14123
14124 ins_pipe(fp_l2d);
14125
14126 %}
14127
14128 // ============================================================================
14129 // clearing of an array
14130
14131 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14132 %{
14133 match(Set dummy (ClearArray cnt base));
14134 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14135
14136 ins_cost(4 * INSN_COST);
14137 format %{ "ClearArray $cnt, $base" %}
14138
14139 ins_encode %{
14140 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14141 if (tpc == nullptr) {
14142 ciEnv::current()->record_failure("CodeCache is full");
14143 return;
14144 }
14145 %}
14146
14147 ins_pipe(pipe_class_memory);
14148 %}
14149
14150 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14151 %{
14152 predicate((uint64_t)n->in(2)->get_long()
14153 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14154 match(Set dummy (ClearArray cnt base));
14155 effect(TEMP temp, USE_KILL base, KILL cr);
14156
14157 ins_cost(4 * INSN_COST);
14158 format %{ "ClearArray $cnt, $base" %}
14159
14160 ins_encode %{
14161 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14162 if (tpc == nullptr) {
14163 ciEnv::current()->record_failure("CodeCache is full");
14164 return;
14165 }
14166 %}
14167
14168 ins_pipe(pipe_class_memory);
14169 %}
14170
14171 // ============================================================================
14172 // Overflow Math Instructions
14173
14174 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14175 %{
14176 match(Set cr (OverflowAddI op1 op2));
14177
14178 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14179 ins_cost(INSN_COST);
14180 ins_encode %{
14181 __ cmnw($op1$$Register, $op2$$Register);
14182 %}
14183
14184 ins_pipe(icmp_reg_reg);
14185 %}
14186
14187 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14188 %{
14189 match(Set cr (OverflowAddI op1 op2));
14190
14191 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14192 ins_cost(INSN_COST);
14193 ins_encode %{
14194 __ cmnw($op1$$Register, $op2$$constant);
14195 %}
14196
14197 ins_pipe(icmp_reg_imm);
14198 %}
14199
14200 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14201 %{
14202 match(Set cr (OverflowAddL op1 op2));
14203
14204 format %{ "cmn $op1, $op2\t# overflow check long" %}
14205 ins_cost(INSN_COST);
14206 ins_encode %{
14207 __ cmn($op1$$Register, $op2$$Register);
14208 %}
14209
14210 ins_pipe(icmp_reg_reg);
14211 %}
14212
14213 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14214 %{
14215 match(Set cr (OverflowAddL op1 op2));
14216
14217 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14218 ins_cost(INSN_COST);
14219 ins_encode %{
14220 __ adds(zr, $op1$$Register, $op2$$constant);
14221 %}
14222
14223 ins_pipe(icmp_reg_imm);
14224 %}
14225
14226 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14227 %{
14228 match(Set cr (OverflowSubI op1 op2));
14229
14230 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14231 ins_cost(INSN_COST);
14232 ins_encode %{
14233 __ cmpw($op1$$Register, $op2$$Register);
14234 %}
14235
14236 ins_pipe(icmp_reg_reg);
14237 %}
14238
14239 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14240 %{
14241 match(Set cr (OverflowSubI op1 op2));
14242
14243 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14244 ins_cost(INSN_COST);
14245 ins_encode %{
14246 __ cmpw($op1$$Register, $op2$$constant);
14247 %}
14248
14249 ins_pipe(icmp_reg_imm);
14250 %}
14251
14252 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14253 %{
14254 match(Set cr (OverflowSubL op1 op2));
14255
14256 format %{ "cmp $op1, $op2\t# overflow check long" %}
14257 ins_cost(INSN_COST);
14258 ins_encode %{
14259 __ cmp($op1$$Register, $op2$$Register);
14260 %}
14261
14262 ins_pipe(icmp_reg_reg);
14263 %}
14264
14265 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14266 %{
14267 match(Set cr (OverflowSubL op1 op2));
14268
14269 format %{ "cmp $op1, $op2\t# overflow check long" %}
14270 ins_cost(INSN_COST);
14271 ins_encode %{
14272 __ subs(zr, $op1$$Register, $op2$$constant);
14273 %}
14274
14275 ins_pipe(icmp_reg_imm);
14276 %}
14277
14278 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14279 %{
14280 match(Set cr (OverflowSubI zero op1));
14281
14282 format %{ "cmpw zr, $op1\t# overflow check int" %}
14283 ins_cost(INSN_COST);
14284 ins_encode %{
14285 __ cmpw(zr, $op1$$Register);
14286 %}
14287
14288 ins_pipe(icmp_reg_imm);
14289 %}
14290
14291 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14292 %{
14293 match(Set cr (OverflowSubL zero op1));
14294
14295 format %{ "cmp zr, $op1\t# overflow check long" %}
14296 ins_cost(INSN_COST);
14297 ins_encode %{
14298 __ cmp(zr, $op1$$Register);
14299 %}
14300
14301 ins_pipe(icmp_reg_imm);
14302 %}
14303
14304 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14305 %{
14306 match(Set cr (OverflowMulI op1 op2));
14307
14308 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14309 "cmp rscratch1, rscratch1, sxtw\n\t"
14310 "movw rscratch1, #0x80000000\n\t"
14311 "cselw rscratch1, rscratch1, zr, NE\n\t"
14312 "cmpw rscratch1, #1" %}
14313 ins_cost(5 * INSN_COST);
14314 ins_encode %{
14315 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14316 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14317 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14318 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14319 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14320 %}
14321
14322 ins_pipe(pipe_slow);
14323 %}
14324
14325 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14326 %{
14327 match(If cmp (OverflowMulI op1 op2));
14328 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14329 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14330 effect(USE labl, KILL cr);
14331
14332 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14333 "cmp rscratch1, rscratch1, sxtw\n\t"
14334 "b$cmp $labl" %}
14335 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14336 ins_encode %{
14337 Label* L = $labl$$label;
14338 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14339 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14340 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14341 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14342 %}
14343
14344 ins_pipe(pipe_serial);
14345 %}
14346
14347 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14348 %{
14349 match(Set cr (OverflowMulL op1 op2));
14350
14351 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14352 "smulh rscratch2, $op1, $op2\n\t"
14353 "cmp rscratch2, rscratch1, ASR #63\n\t"
14354 "movw rscratch1, #0x80000000\n\t"
14355 "cselw rscratch1, rscratch1, zr, NE\n\t"
14356 "cmpw rscratch1, #1" %}
14357 ins_cost(6 * INSN_COST);
14358 ins_encode %{
14359 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14360 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14361 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14362 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14363 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14364 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14365 %}
14366
14367 ins_pipe(pipe_slow);
14368 %}
14369
14370 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14371 %{
14372 match(If cmp (OverflowMulL op1 op2));
14373 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14374 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14375 effect(USE labl, KILL cr);
14376
14377 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14378 "smulh rscratch2, $op1, $op2\n\t"
14379 "cmp rscratch2, rscratch1, ASR #63\n\t"
14380 "b$cmp $labl" %}
14381 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14382 ins_encode %{
14383 Label* L = $labl$$label;
14384 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14385 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14386 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14387 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14388 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14389 %}
14390
14391 ins_pipe(pipe_serial);
14392 %}
14393
14394 // ============================================================================
14395 // Compare Instructions
14396
14397 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14398 %{
14399 match(Set cr (CmpI op1 op2));
14400
14401 effect(DEF cr, USE op1, USE op2);
14402
14403 ins_cost(INSN_COST);
14404 format %{ "cmpw $op1, $op2" %}
14405
14406 ins_encode(aarch64_enc_cmpw(op1, op2));
14407
14408 ins_pipe(icmp_reg_reg);
14409 %}
14410
14411 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14412 %{
14413 match(Set cr (CmpI op1 zero));
14414
14415 effect(DEF cr, USE op1);
14416
14417 ins_cost(INSN_COST);
14418 format %{ "cmpw $op1, 0" %}
14419
14420 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14421
14422 ins_pipe(icmp_reg_imm);
14423 %}
14424
14425 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14426 %{
14427 match(Set cr (CmpI op1 op2));
14428
14429 effect(DEF cr, USE op1);
14430
14431 ins_cost(INSN_COST);
14432 format %{ "cmpw $op1, $op2" %}
14433
14434 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14435
14436 ins_pipe(icmp_reg_imm);
14437 %}
14438
14439 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14440 %{
14441 match(Set cr (CmpI op1 op2));
14442
14443 effect(DEF cr, USE op1);
14444
14445 ins_cost(INSN_COST * 2);
14446 format %{ "cmpw $op1, $op2" %}
14447
14448 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14449
14450 ins_pipe(icmp_reg_imm);
14451 %}
14452
14453 // Unsigned compare Instructions; really, same as signed compare
14454 // except it should only be used to feed an If or a CMovI which takes a
14455 // cmpOpU.
14456
14457 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14458 %{
14459 match(Set cr (CmpU op1 op2));
14460
14461 effect(DEF cr, USE op1, USE op2);
14462
14463 ins_cost(INSN_COST);
14464 format %{ "cmpw $op1, $op2\t# unsigned" %}
14465
14466 ins_encode(aarch64_enc_cmpw(op1, op2));
14467
14468 ins_pipe(icmp_reg_reg);
14469 %}
14470
14471 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14472 %{
14473 match(Set cr (CmpU op1 zero));
14474
14475 effect(DEF cr, USE op1);
14476
14477 ins_cost(INSN_COST);
14478 format %{ "cmpw $op1, #0\t# unsigned" %}
14479
14480 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14481
14482 ins_pipe(icmp_reg_imm);
14483 %}
14484
14485 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14486 %{
14487 match(Set cr (CmpU op1 op2));
14488
14489 effect(DEF cr, USE op1);
14490
14491 ins_cost(INSN_COST);
14492 format %{ "cmpw $op1, $op2\t# unsigned" %}
14493
14494 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14495
14496 ins_pipe(icmp_reg_imm);
14497 %}
14498
14499 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14500 %{
14501 match(Set cr (CmpU op1 op2));
14502
14503 effect(DEF cr, USE op1);
14504
14505 ins_cost(INSN_COST * 2);
14506 format %{ "cmpw $op1, $op2\t# unsigned" %}
14507
14508 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14509
14510 ins_pipe(icmp_reg_imm);
14511 %}
14512
14513 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14514 %{
14515 match(Set cr (CmpL op1 op2));
14516
14517 effect(DEF cr, USE op1, USE op2);
14518
14519 ins_cost(INSN_COST);
14520 format %{ "cmp $op1, $op2" %}
14521
14522 ins_encode(aarch64_enc_cmp(op1, op2));
14523
14524 ins_pipe(icmp_reg_reg);
14525 %}
14526
14527 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14528 %{
14529 match(Set cr (CmpL op1 zero));
14530
14531 effect(DEF cr, USE op1);
14532
14533 ins_cost(INSN_COST);
14534 format %{ "tst $op1" %}
14535
14536 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14537
14538 ins_pipe(icmp_reg_imm);
14539 %}
14540
14541 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14542 %{
14543 match(Set cr (CmpL op1 op2));
14544
14545 effect(DEF cr, USE op1);
14546
14547 ins_cost(INSN_COST);
14548 format %{ "cmp $op1, $op2" %}
14549
14550 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14551
14552 ins_pipe(icmp_reg_imm);
14553 %}
14554
14555 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14556 %{
14557 match(Set cr (CmpL op1 op2));
14558
14559 effect(DEF cr, USE op1);
14560
14561 ins_cost(INSN_COST * 2);
14562 format %{ "cmp $op1, $op2" %}
14563
14564 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14565
14566 ins_pipe(icmp_reg_imm);
14567 %}
14568
14569 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14570 %{
14571 match(Set cr (CmpUL op1 op2));
14572
14573 effect(DEF cr, USE op1, USE op2);
14574
14575 ins_cost(INSN_COST);
14576 format %{ "cmp $op1, $op2" %}
14577
14578 ins_encode(aarch64_enc_cmp(op1, op2));
14579
14580 ins_pipe(icmp_reg_reg);
14581 %}
14582
14583 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14584 %{
14585 match(Set cr (CmpUL op1 zero));
14586
14587 effect(DEF cr, USE op1);
14588
14589 ins_cost(INSN_COST);
14590 format %{ "tst $op1" %}
14591
14592 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14593
14594 ins_pipe(icmp_reg_imm);
14595 %}
14596
14597 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14598 %{
14599 match(Set cr (CmpUL op1 op2));
14600
14601 effect(DEF cr, USE op1);
14602
14603 ins_cost(INSN_COST);
14604 format %{ "cmp $op1, $op2" %}
14605
14606 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14607
14608 ins_pipe(icmp_reg_imm);
14609 %}
14610
14611 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14612 %{
14613 match(Set cr (CmpUL op1 op2));
14614
14615 effect(DEF cr, USE op1);
14616
14617 ins_cost(INSN_COST * 2);
14618 format %{ "cmp $op1, $op2" %}
14619
14620 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14621
14622 ins_pipe(icmp_reg_imm);
14623 %}
14624
14625 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14626 %{
14627 match(Set cr (CmpP op1 op2));
14628
14629 effect(DEF cr, USE op1, USE op2);
14630
14631 ins_cost(INSN_COST);
14632 format %{ "cmp $op1, $op2\t // ptr" %}
14633
14634 ins_encode(aarch64_enc_cmpp(op1, op2));
14635
14636 ins_pipe(icmp_reg_reg);
14637 %}
14638
14639 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14640 %{
14641 match(Set cr (CmpN op1 op2));
14642
14643 effect(DEF cr, USE op1, USE op2);
14644
14645 ins_cost(INSN_COST);
14646 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14647
14648 ins_encode(aarch64_enc_cmpn(op1, op2));
14649
14650 ins_pipe(icmp_reg_reg);
14651 %}
14652
14653 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14654 %{
14655 match(Set cr (CmpP op1 zero));
14656
14657 effect(DEF cr, USE op1, USE zero);
14658
14659 ins_cost(INSN_COST);
14660 format %{ "cmp $op1, 0\t // ptr" %}
14661
14662 ins_encode(aarch64_enc_testp(op1));
14663
14664 ins_pipe(icmp_reg_imm);
14665 %}
14666
14667 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14668 %{
14669 match(Set cr (CmpN op1 zero));
14670
14671 effect(DEF cr, USE op1, USE zero);
14672
14673 ins_cost(INSN_COST);
14674 format %{ "cmp $op1, 0\t // compressed ptr" %}
14675
14676 ins_encode(aarch64_enc_testn(op1));
14677
14678 ins_pipe(icmp_reg_imm);
14679 %}
14680
14681 // FP comparisons
14682 //
14683 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14684 // using normal cmpOp. See declaration of rFlagsReg for details.
14685
14686 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14687 %{
14688 match(Set cr (CmpF src1 src2));
14689
14690 ins_cost(3 * INSN_COST);
14691 format %{ "fcmps $src1, $src2" %}
14692
14693 ins_encode %{
14694 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14695 %}
14696
14697 ins_pipe(pipe_class_compare);
14698 %}
14699
14700 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14701 %{
14702 match(Set cr (CmpF src1 src2));
14703
14704 ins_cost(3 * INSN_COST);
14705 format %{ "fcmps $src1, 0.0" %}
14706
14707 ins_encode %{
14708 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14709 %}
14710
14711 ins_pipe(pipe_class_compare);
14712 %}
14713 // FROM HERE
14714
14715 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14716 %{
14717 match(Set cr (CmpD src1 src2));
14718
14719 ins_cost(3 * INSN_COST);
14720 format %{ "fcmpd $src1, $src2" %}
14721
14722 ins_encode %{
14723 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14724 %}
14725
14726 ins_pipe(pipe_class_compare);
14727 %}
14728
14729 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14730 %{
14731 match(Set cr (CmpD src1 src2));
14732
14733 ins_cost(3 * INSN_COST);
14734 format %{ "fcmpd $src1, 0.0" %}
14735
14736 ins_encode %{
14737 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14738 %}
14739
14740 ins_pipe(pipe_class_compare);
14741 %}
14742
14743 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14744 %{
14745 match(Set dst (CmpF3 src1 src2));
14746 effect(KILL cr);
14747
14748 ins_cost(5 * INSN_COST);
14749 format %{ "fcmps $src1, $src2\n\t"
14750 "csinvw($dst, zr, zr, eq\n\t"
14751 "csnegw($dst, $dst, $dst, lt)"
14752 %}
14753
14754 ins_encode %{
14755 Label done;
14756 FloatRegister s1 = as_FloatRegister($src1$$reg);
14757 FloatRegister s2 = as_FloatRegister($src2$$reg);
14758 Register d = as_Register($dst$$reg);
14759 __ fcmps(s1, s2);
14760 // installs 0 if EQ else -1
14761 __ csinvw(d, zr, zr, Assembler::EQ);
14762 // keeps -1 if less or unordered else installs 1
14763 __ csnegw(d, d, d, Assembler::LT);
14764 __ bind(done);
14765 %}
14766
14767 ins_pipe(pipe_class_default);
14768
14769 %}
14770
14771 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14772 %{
14773 match(Set dst (CmpD3 src1 src2));
14774 effect(KILL cr);
14775
14776 ins_cost(5 * INSN_COST);
14777 format %{ "fcmpd $src1, $src2\n\t"
14778 "csinvw($dst, zr, zr, eq\n\t"
14779 "csnegw($dst, $dst, $dst, lt)"
14780 %}
14781
14782 ins_encode %{
14783 Label done;
14784 FloatRegister s1 = as_FloatRegister($src1$$reg);
14785 FloatRegister s2 = as_FloatRegister($src2$$reg);
14786 Register d = as_Register($dst$$reg);
14787 __ fcmpd(s1, s2);
14788 // installs 0 if EQ else -1
14789 __ csinvw(d, zr, zr, Assembler::EQ);
14790 // keeps -1 if less or unordered else installs 1
14791 __ csnegw(d, d, d, Assembler::LT);
14792 __ bind(done);
14793 %}
14794 ins_pipe(pipe_class_default);
14795
14796 %}
14797
14798 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14799 %{
14800 match(Set dst (CmpF3 src1 zero));
14801 effect(KILL cr);
14802
14803 ins_cost(5 * INSN_COST);
14804 format %{ "fcmps $src1, 0.0\n\t"
14805 "csinvw($dst, zr, zr, eq\n\t"
14806 "csnegw($dst, $dst, $dst, lt)"
14807 %}
14808
14809 ins_encode %{
14810 Label done;
14811 FloatRegister s1 = as_FloatRegister($src1$$reg);
14812 Register d = as_Register($dst$$reg);
14813 __ fcmps(s1, 0.0);
14814 // installs 0 if EQ else -1
14815 __ csinvw(d, zr, zr, Assembler::EQ);
14816 // keeps -1 if less or unordered else installs 1
14817 __ csnegw(d, d, d, Assembler::LT);
14818 __ bind(done);
14819 %}
14820
14821 ins_pipe(pipe_class_default);
14822
14823 %}
14824
14825 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14826 %{
14827 match(Set dst (CmpD3 src1 zero));
14828 effect(KILL cr);
14829
14830 ins_cost(5 * INSN_COST);
14831 format %{ "fcmpd $src1, 0.0\n\t"
14832 "csinvw($dst, zr, zr, eq\n\t"
14833 "csnegw($dst, $dst, $dst, lt)"
14834 %}
14835
14836 ins_encode %{
14837 Label done;
14838 FloatRegister s1 = as_FloatRegister($src1$$reg);
14839 Register d = as_Register($dst$$reg);
14840 __ fcmpd(s1, 0.0);
14841 // installs 0 if EQ else -1
14842 __ csinvw(d, zr, zr, Assembler::EQ);
14843 // keeps -1 if less or unordered else installs 1
14844 __ csnegw(d, d, d, Assembler::LT);
14845 __ bind(done);
14846 %}
14847 ins_pipe(pipe_class_default);
14848
14849 %}
14850
14851 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14852 %{
14853 match(Set dst (CmpLTMask p q));
14854 effect(KILL cr);
14855
14856 ins_cost(3 * INSN_COST);
14857
14858 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14859 "csetw $dst, lt\n\t"
14860 "subw $dst, zr, $dst"
14861 %}
14862
14863 ins_encode %{
14864 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14865 __ csetw(as_Register($dst$$reg), Assembler::LT);
14866 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14867 %}
14868
14869 ins_pipe(ialu_reg_reg);
14870 %}
14871
14872 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14873 %{
14874 match(Set dst (CmpLTMask src zero));
14875 effect(KILL cr);
14876
14877 ins_cost(INSN_COST);
14878
14879 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14880
14881 ins_encode %{
14882 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14883 %}
14884
14885 ins_pipe(ialu_reg_shift);
14886 %}
14887
14888 // ============================================================================
14889 // Max and Min
14890
14891 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14892
14893 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14894 %{
14895 effect(DEF cr, USE src);
14896 ins_cost(INSN_COST);
14897 format %{ "cmpw $src, 0" %}
14898
14899 ins_encode %{
14900 __ cmpw($src$$Register, 0);
14901 %}
14902 ins_pipe(icmp_reg_imm);
14903 %}
14904
14905 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14906 %{
14907 match(Set dst (MinI src1 src2));
14908 ins_cost(INSN_COST * 3);
14909
14910 expand %{
14911 rFlagsReg cr;
14912 compI_reg_reg(cr, src1, src2);
14913 cmovI_reg_reg_lt(dst, src1, src2, cr);
14914 %}
14915 %}
14916
14917 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14918 %{
14919 match(Set dst (MaxI src1 src2));
14920 ins_cost(INSN_COST * 3);
14921
14922 expand %{
14923 rFlagsReg cr;
14924 compI_reg_reg(cr, src1, src2);
14925 cmovI_reg_reg_gt(dst, src1, src2, cr);
14926 %}
14927 %}
14928
14929
14930 // ============================================================================
14931 // Branch Instructions
14932
14933 // Direct Branch.
14934 instruct branch(label lbl)
14935 %{
14936 match(Goto);
14937
14938 effect(USE lbl);
14939
14940 ins_cost(BRANCH_COST);
14941 format %{ "b $lbl" %}
14942
14943 ins_encode(aarch64_enc_b(lbl));
14944
14945 ins_pipe(pipe_branch);
14946 %}
14947
14948 // Conditional Near Branch
14949 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14950 %{
14951 // Same match rule as `branchConFar'.
14952 match(If cmp cr);
14953
14954 effect(USE lbl);
14955
14956 ins_cost(BRANCH_COST);
14957 // If set to 1 this indicates that the current instruction is a
14958 // short variant of a long branch. This avoids using this
14959 // instruction in first-pass matching. It will then only be used in
14960 // the `Shorten_branches' pass.
14961 // ins_short_branch(1);
14962 format %{ "b$cmp $lbl" %}
14963
14964 ins_encode(aarch64_enc_br_con(cmp, lbl));
14965
14966 ins_pipe(pipe_branch_cond);
14967 %}
14968
14969 // Conditional Near Branch Unsigned
14970 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14971 %{
14972 // Same match rule as `branchConFar'.
14973 match(If cmp cr);
14974
14975 effect(USE lbl);
14976
14977 ins_cost(BRANCH_COST);
14978 // If set to 1 this indicates that the current instruction is a
14979 // short variant of a long branch. This avoids using this
14980 // instruction in first-pass matching. It will then only be used in
14981 // the `Shorten_branches' pass.
14982 // ins_short_branch(1);
14983 format %{ "b$cmp $lbl\t# unsigned" %}
14984
14985 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14986
14987 ins_pipe(pipe_branch_cond);
14988 %}
14989
14990 // Make use of CBZ and CBNZ. These instructions, as well as being
14991 // shorter than (cmp; branch), have the additional benefit of not
14992 // killing the flags.
14993
14994 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14995 match(If cmp (CmpI op1 op2));
14996 effect(USE labl);
14997
14998 ins_cost(BRANCH_COST);
14999 format %{ "cbw$cmp $op1, $labl" %}
15000 ins_encode %{
15001 Label* L = $labl$$label;
15002 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15003 if (cond == Assembler::EQ)
15004 __ cbzw($op1$$Register, *L);
15005 else
15006 __ cbnzw($op1$$Register, *L);
15007 %}
15008 ins_pipe(pipe_cmp_branch);
15009 %}
15010
15011 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15012 match(If cmp (CmpL op1 op2));
15013 effect(USE labl);
15014
15015 ins_cost(BRANCH_COST);
15016 format %{ "cb$cmp $op1, $labl" %}
15017 ins_encode %{
15018 Label* L = $labl$$label;
15019 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15020 if (cond == Assembler::EQ)
15021 __ cbz($op1$$Register, *L);
15022 else
15023 __ cbnz($op1$$Register, *L);
15024 %}
15025 ins_pipe(pipe_cmp_branch);
15026 %}
15027
15028 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15029 match(If cmp (CmpP op1 op2));
15030 effect(USE labl);
15031
15032 ins_cost(BRANCH_COST);
15033 format %{ "cb$cmp $op1, $labl" %}
15034 ins_encode %{
15035 Label* L = $labl$$label;
15036 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15037 if (cond == Assembler::EQ)
15038 __ cbz($op1$$Register, *L);
15039 else
15040 __ cbnz($op1$$Register, *L);
15041 %}
15042 ins_pipe(pipe_cmp_branch);
15043 %}
15044
15045 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15046 match(If cmp (CmpN op1 op2));
15047 effect(USE labl);
15048
15049 ins_cost(BRANCH_COST);
15050 format %{ "cbw$cmp $op1, $labl" %}
15051 ins_encode %{
15052 Label* L = $labl$$label;
15053 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15054 if (cond == Assembler::EQ)
15055 __ cbzw($op1$$Register, *L);
15056 else
15057 __ cbnzw($op1$$Register, *L);
15058 %}
15059 ins_pipe(pipe_cmp_branch);
15060 %}
15061
15062 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15063 match(If cmp (CmpP (DecodeN oop) zero));
15064 effect(USE labl);
15065
15066 ins_cost(BRANCH_COST);
15067 format %{ "cb$cmp $oop, $labl" %}
15068 ins_encode %{
15069 Label* L = $labl$$label;
15070 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15071 if (cond == Assembler::EQ)
15072 __ cbzw($oop$$Register, *L);
15073 else
15074 __ cbnzw($oop$$Register, *L);
15075 %}
15076 ins_pipe(pipe_cmp_branch);
15077 %}
15078
15079 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15080 match(If cmp (CmpU op1 op2));
15081 effect(USE labl);
15082
15083 ins_cost(BRANCH_COST);
15084 format %{ "cbw$cmp $op1, $labl" %}
15085 ins_encode %{
15086 Label* L = $labl$$label;
15087 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15088 if (cond == Assembler::EQ || cond == Assembler::LS) {
15089 __ cbzw($op1$$Register, *L);
15090 } else {
15091 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15092 __ cbnzw($op1$$Register, *L);
15093 }
15094 %}
15095 ins_pipe(pipe_cmp_branch);
15096 %}
15097
15098 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15099 match(If cmp (CmpUL op1 op2));
15100 effect(USE labl);
15101
15102 ins_cost(BRANCH_COST);
15103 format %{ "cb$cmp $op1, $labl" %}
15104 ins_encode %{
15105 Label* L = $labl$$label;
15106 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15107 if (cond == Assembler::EQ || cond == Assembler::LS) {
15108 __ cbz($op1$$Register, *L);
15109 } else {
15110 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15111 __ cbnz($op1$$Register, *L);
15112 }
15113 %}
15114 ins_pipe(pipe_cmp_branch);
15115 %}
15116
15117 // Test bit and Branch
15118
15119 // Patterns for short (< 32KiB) variants
15120 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15121 match(If cmp (CmpL op1 op2));
15122 effect(USE labl);
15123
15124 ins_cost(BRANCH_COST);
15125 format %{ "cb$cmp $op1, $labl # long" %}
15126 ins_encode %{
15127 Label* L = $labl$$label;
15128 Assembler::Condition cond =
15129 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15130 __ tbr(cond, $op1$$Register, 63, *L);
15131 %}
15132 ins_pipe(pipe_cmp_branch);
15133 ins_short_branch(1);
15134 %}
15135
15136 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15137 match(If cmp (CmpI op1 op2));
15138 effect(USE labl);
15139
15140 ins_cost(BRANCH_COST);
15141 format %{ "cb$cmp $op1, $labl # int" %}
15142 ins_encode %{
15143 Label* L = $labl$$label;
15144 Assembler::Condition cond =
15145 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15146 __ tbr(cond, $op1$$Register, 31, *L);
15147 %}
15148 ins_pipe(pipe_cmp_branch);
15149 ins_short_branch(1);
15150 %}
15151
15152 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15153 match(If cmp (CmpL (AndL op1 op2) op3));
15154 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15155 effect(USE labl);
15156
15157 ins_cost(BRANCH_COST);
15158 format %{ "tb$cmp $op1, $op2, $labl" %}
15159 ins_encode %{
15160 Label* L = $labl$$label;
15161 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15162 int bit = exact_log2_long($op2$$constant);
15163 __ tbr(cond, $op1$$Register, bit, *L);
15164 %}
15165 ins_pipe(pipe_cmp_branch);
15166 ins_short_branch(1);
15167 %}
15168
15169 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15170 match(If cmp (CmpI (AndI op1 op2) op3));
15171 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15172 effect(USE labl);
15173
15174 ins_cost(BRANCH_COST);
15175 format %{ "tb$cmp $op1, $op2, $labl" %}
15176 ins_encode %{
15177 Label* L = $labl$$label;
15178 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15179 int bit = exact_log2((juint)$op2$$constant);
15180 __ tbr(cond, $op1$$Register, bit, *L);
15181 %}
15182 ins_pipe(pipe_cmp_branch);
15183 ins_short_branch(1);
15184 %}
15185
15186 // And far variants
15187 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15188 match(If cmp (CmpL op1 op2));
15189 effect(USE labl);
15190
15191 ins_cost(BRANCH_COST);
15192 format %{ "cb$cmp $op1, $labl # long" %}
15193 ins_encode %{
15194 Label* L = $labl$$label;
15195 Assembler::Condition cond =
15196 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15197 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15198 %}
15199 ins_pipe(pipe_cmp_branch);
15200 %}
15201
15202 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15203 match(If cmp (CmpI op1 op2));
15204 effect(USE labl);
15205
15206 ins_cost(BRANCH_COST);
15207 format %{ "cb$cmp $op1, $labl # int" %}
15208 ins_encode %{
15209 Label* L = $labl$$label;
15210 Assembler::Condition cond =
15211 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15212 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15213 %}
15214 ins_pipe(pipe_cmp_branch);
15215 %}
15216
15217 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15218 match(If cmp (CmpL (AndL op1 op2) op3));
15219 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15220 effect(USE labl);
15221
15222 ins_cost(BRANCH_COST);
15223 format %{ "tb$cmp $op1, $op2, $labl" %}
15224 ins_encode %{
15225 Label* L = $labl$$label;
15226 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15227 int bit = exact_log2_long($op2$$constant);
15228 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15229 %}
15230 ins_pipe(pipe_cmp_branch);
15231 %}
15232
15233 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15234 match(If cmp (CmpI (AndI op1 op2) op3));
15235 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15236 effect(USE labl);
15237
15238 ins_cost(BRANCH_COST);
15239 format %{ "tb$cmp $op1, $op2, $labl" %}
15240 ins_encode %{
15241 Label* L = $labl$$label;
15242 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15243 int bit = exact_log2((juint)$op2$$constant);
15244 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15245 %}
15246 ins_pipe(pipe_cmp_branch);
15247 %}
15248
15249 // Test bits
15250
15251 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15252 match(Set cr (CmpL (AndL op1 op2) op3));
15253 predicate(Assembler::operand_valid_for_logical_immediate
15254 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15255
15256 ins_cost(INSN_COST);
15257 format %{ "tst $op1, $op2 # long" %}
15258 ins_encode %{
15259 __ tst($op1$$Register, $op2$$constant);
15260 %}
15261 ins_pipe(ialu_reg_reg);
15262 %}
15263
15264 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15265 match(Set cr (CmpI (AndI op1 op2) op3));
15266 predicate(Assembler::operand_valid_for_logical_immediate
15267 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15268
15269 ins_cost(INSN_COST);
15270 format %{ "tst $op1, $op2 # int" %}
15271 ins_encode %{
15272 __ tstw($op1$$Register, $op2$$constant);
15273 %}
15274 ins_pipe(ialu_reg_reg);
15275 %}
15276
15277 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15278 match(Set cr (CmpL (AndL op1 op2) op3));
15279
15280 ins_cost(INSN_COST);
15281 format %{ "tst $op1, $op2 # long" %}
15282 ins_encode %{
15283 __ tst($op1$$Register, $op2$$Register);
15284 %}
15285 ins_pipe(ialu_reg_reg);
15286 %}
15287
15288 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15289 match(Set cr (CmpI (AndI op1 op2) op3));
15290
15291 ins_cost(INSN_COST);
15292 format %{ "tstw $op1, $op2 # int" %}
15293 ins_encode %{
15294 __ tstw($op1$$Register, $op2$$Register);
15295 %}
15296 ins_pipe(ialu_reg_reg);
15297 %}
15298
15299
15300 // Conditional Far Branch
15301 // Conditional Far Branch Unsigned
15302 // TODO: fixme
15303
15304 // counted loop end branch near
15305 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15306 %{
15307 match(CountedLoopEnd cmp cr);
15308
15309 effect(USE lbl);
15310
15311 ins_cost(BRANCH_COST);
15312 // short variant.
15313 // ins_short_branch(1);
15314 format %{ "b$cmp $lbl \t// counted loop end" %}
15315
15316 ins_encode(aarch64_enc_br_con(cmp, lbl));
15317
15318 ins_pipe(pipe_branch);
15319 %}
15320
15321 // counted loop end branch far
15322 // TODO: fixme
15323
15324 // ============================================================================
15325 // inlined locking and unlocking
15326
15327 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15328 %{
15329 match(Set cr (FastLock object box));
15330 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15331
15332 ins_cost(5 * INSN_COST);
15333 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15334
15335 ins_encode %{
15336 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15337 %}
15338
15339 ins_pipe(pipe_serial);
15340 %}
15341
15342 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15343 %{
15344 match(Set cr (FastUnlock object box));
15345 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15346
15347 ins_cost(5 * INSN_COST);
15348 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15349
15350 ins_encode %{
15351 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15352 %}
15353
15354 ins_pipe(pipe_serial);
15355 %}
15356
15357 // ============================================================================
15358 // Safepoint Instructions
15359
15360 // TODO
15361 // provide a near and far version of this code
15362
15363 instruct safePoint(rFlagsReg cr, iRegP poll)
15364 %{
15365 match(SafePoint poll);
15366 effect(KILL cr);
15367
15368 format %{
15369 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15370 %}
15371 ins_encode %{
15372 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15373 %}
15374 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15375 %}
15376
15377
15378 // ============================================================================
15379 // Procedure Call/Return Instructions
15380
15381 // Call Java Static Instruction
15382
15383 instruct CallStaticJavaDirect(method meth)
15384 %{
15385 match(CallStaticJava);
15386
15387 effect(USE meth);
15388
15389 ins_cost(CALL_COST);
15390
15391 format %{ "call,static $meth \t// ==> " %}
15392
15393 ins_encode(aarch64_enc_java_static_call(meth),
15394 aarch64_enc_call_epilog);
15395
15396 ins_pipe(pipe_class_call);
15397 %}
15398
15399 // TO HERE
15400
15401 // Call Java Dynamic Instruction
15402 instruct CallDynamicJavaDirect(method meth)
15403 %{
15404 match(CallDynamicJava);
15405
15406 effect(USE meth);
15407
15408 ins_cost(CALL_COST);
15409
15410 format %{ "CALL,dynamic $meth \t// ==> " %}
15411
15412 ins_encode(aarch64_enc_java_dynamic_call(meth),
15413 aarch64_enc_call_epilog);
15414
15415 ins_pipe(pipe_class_call);
15416 %}
15417
15418 // Call Runtime Instruction
15419
15420 instruct CallRuntimeDirect(method meth)
15421 %{
15422 match(CallRuntime);
15423
15424 effect(USE meth);
15425
15426 ins_cost(CALL_COST);
15427
15428 format %{ "CALL, runtime $meth" %}
15429
15430 ins_encode( aarch64_enc_java_to_runtime(meth) );
15431
15432 ins_pipe(pipe_class_call);
15433 %}
15434
15435 // Call Runtime Instruction
15436
15437 instruct CallLeafDirect(method meth)
15438 %{
15439 match(CallLeaf);
15440
15441 effect(USE meth);
15442
15443 ins_cost(CALL_COST);
15444
15445 format %{ "CALL, runtime leaf $meth" %}
15446
15447 ins_encode( aarch64_enc_java_to_runtime(meth) );
15448
15449 ins_pipe(pipe_class_call);
15450 %}
15451
15452 // Call Runtime Instruction without safepoint and with vector arguments
15453 instruct CallLeafDirectVector(method meth)
15454 %{
15455 match(CallLeafVector);
15456
15457 effect(USE meth);
15458
15459 ins_cost(CALL_COST);
15460
15461 format %{ "CALL, runtime leaf vector $meth" %}
15462
15463 ins_encode(aarch64_enc_java_to_runtime(meth));
15464
15465 ins_pipe(pipe_class_call);
15466 %}
15467
15468 // Call Runtime Instruction
15469
15470 instruct CallLeafNoFPDirect(method meth)
15471 %{
15472 match(CallLeafNoFP);
15473
15474 effect(USE meth);
15475
15476 ins_cost(CALL_COST);
15477
15478 format %{ "CALL, runtime leaf nofp $meth" %}
15479
15480 ins_encode( aarch64_enc_java_to_runtime(meth) );
15481
15482 ins_pipe(pipe_class_call);
15483 %}
15484
15485 // Tail Call; Jump from runtime stub to Java code.
15486 // Also known as an 'interprocedural jump'.
15487 // Target of jump will eventually return to caller.
15488 // TailJump below removes the return address.
15489 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15490 // emitted just above the TailCall which has reset rfp to the caller state.
15491 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15492 %{
15493 match(TailCall jump_target method_ptr);
15494
15495 ins_cost(CALL_COST);
15496
15497 format %{ "br $jump_target\t# $method_ptr holds method" %}
15498
15499 ins_encode(aarch64_enc_tail_call(jump_target));
15500
15501 ins_pipe(pipe_class_call);
15502 %}
15503
15504 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15505 %{
15506 match(TailJump jump_target ex_oop);
15507
15508 ins_cost(CALL_COST);
15509
15510 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15511
15512 ins_encode(aarch64_enc_tail_jmp(jump_target));
15513
15514 ins_pipe(pipe_class_call);
15515 %}
15516
15517 // Forward exception.
15518 instruct ForwardExceptionjmp()
15519 %{
15520 match(ForwardException);
15521 ins_cost(CALL_COST);
15522
15523 format %{ "b forward_exception_stub" %}
15524 ins_encode %{
15525 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15526 %}
15527 ins_pipe(pipe_class_call);
15528 %}
15529
15530 // Create exception oop: created by stack-crawling runtime code.
15531 // Created exception is now available to this handler, and is setup
15532 // just prior to jumping to this handler. No code emitted.
15533 // TODO check
15534 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15535 instruct CreateException(iRegP_R0 ex_oop)
15536 %{
15537 match(Set ex_oop (CreateEx));
15538
15539 format %{ " -- \t// exception oop; no code emitted" %}
15540
15541 size(0);
15542
15543 ins_encode( /*empty*/ );
15544
15545 ins_pipe(pipe_class_empty);
15546 %}
15547
15548 // Rethrow exception: The exception oop will come in the first
15549 // argument position. Then JUMP (not call) to the rethrow stub code.
15550 instruct RethrowException() %{
15551 match(Rethrow);
15552 ins_cost(CALL_COST);
15553
15554 format %{ "b rethrow_stub" %}
15555
15556 ins_encode( aarch64_enc_rethrow() );
15557
15558 ins_pipe(pipe_class_call);
15559 %}
15560
15561
15562 // Return Instruction
15563 // epilog node loads ret address into lr as part of frame pop
15564 instruct Ret()
15565 %{
15566 match(Return);
15567
15568 format %{ "ret\t// return register" %}
15569
15570 ins_encode( aarch64_enc_ret() );
15571
15572 ins_pipe(pipe_branch);
15573 %}
15574
15575 // Die now.
15576 instruct ShouldNotReachHere() %{
15577 match(Halt);
15578
15579 ins_cost(CALL_COST);
15580 format %{ "ShouldNotReachHere" %}
15581
15582 ins_encode %{
15583 if (is_reachable()) {
15584 const char* str = __ code_string(_halt_reason);
15585 __ stop(str);
15586 }
15587 %}
15588
15589 ins_pipe(pipe_class_default);
15590 %}
15591
15592 // ============================================================================
15593 // Partial Subtype Check
15594 //
15595 // superklass array for an instance of the superklass. Set a hidden
15596 // internal cache on a hit (cache is checked with exposed code in
15597 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15598 // encoding ALSO sets flags.
15599
15600 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15601 %{
15602 match(Set result (PartialSubtypeCheck sub super));
15603 predicate(!UseSecondarySupersTable);
15604 effect(KILL cr, KILL temp);
15605
15606 ins_cost(20 * INSN_COST); // slightly larger than the next version
15607 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15608
15609 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15610
15611 opcode(0x1); // Force zero of result reg on hit
15612
15613 ins_pipe(pipe_class_memory);
15614 %}
15615
15616 // Two versions of partialSubtypeCheck, both used when we need to
15617 // search for a super class in the secondary supers array. The first
15618 // is used when we don't know _a priori_ the class being searched
15619 // for. The second, far more common, is used when we do know: this is
15620 // used for instanceof, checkcast, and any case where C2 can determine
15621 // it by constant propagation.
15622
15623 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15624 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15625 rFlagsReg cr)
15626 %{
15627 match(Set result (PartialSubtypeCheck sub super));
15628 predicate(UseSecondarySupersTable);
15629 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15630
15631 ins_cost(10 * INSN_COST); // slightly larger than the next version
15632 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15633
15634 ins_encode %{
15635 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15636 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15637 $vtemp$$FloatRegister,
15638 $result$$Register, /*L_success*/nullptr);
15639 %}
15640
15641 ins_pipe(pipe_class_memory);
15642 %}
15643
15644 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15645 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15646 rFlagsReg cr)
15647 %{
15648 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15649 predicate(UseSecondarySupersTable);
15650 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15651
15652 ins_cost(5 * INSN_COST); // smaller than the next version
15653 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15654
15655 ins_encode %{
15656 bool success = false;
15657 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15658 if (InlineSecondarySupersTest) {
15659 success =
15660 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15661 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15662 $vtemp$$FloatRegister,
15663 $result$$Register,
15664 super_klass_slot);
15665 } else {
15666 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15667 success = (call != nullptr);
15668 }
15669 if (!success) {
15670 ciEnv::current()->record_failure("CodeCache is full");
15671 return;
15672 }
15673 %}
15674
15675 ins_pipe(pipe_class_memory);
15676 %}
15677
15678 // Intrisics for String.compareTo()
15679
15680 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15681 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15682 %{
15683 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15684 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15685 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15686
15687 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15688 ins_encode %{
15689 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15690 __ string_compare($str1$$Register, $str2$$Register,
15691 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15692 $tmp1$$Register, $tmp2$$Register,
15693 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15694 %}
15695 ins_pipe(pipe_class_memory);
15696 %}
15697
15698 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15699 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15700 %{
15701 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15702 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15703 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15704
15705 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15706 ins_encode %{
15707 __ string_compare($str1$$Register, $str2$$Register,
15708 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15709 $tmp1$$Register, $tmp2$$Register,
15710 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15711 %}
15712 ins_pipe(pipe_class_memory);
15713 %}
15714
15715 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15716 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15717 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15718 %{
15719 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15720 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15721 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15722 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15723
15724 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15725 ins_encode %{
15726 __ string_compare($str1$$Register, $str2$$Register,
15727 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15728 $tmp1$$Register, $tmp2$$Register,
15729 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15730 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15731 %}
15732 ins_pipe(pipe_class_memory);
15733 %}
15734
15735 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15736 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15737 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15738 %{
15739 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15740 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15741 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15742 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15743
15744 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15745 ins_encode %{
15746 __ string_compare($str1$$Register, $str2$$Register,
15747 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15748 $tmp1$$Register, $tmp2$$Register,
15749 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15750 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15751 %}
15752 ins_pipe(pipe_class_memory);
15753 %}
15754
15755 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15756 // these string_compare variants as NEON register type for convenience so that the prototype of
15757 // string_compare can be shared with all variants.
15758
15759 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15760 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15761 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15762 pRegGov_P1 pgtmp2, rFlagsReg cr)
15763 %{
15764 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15765 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15766 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15767 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15768
15769 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15770 ins_encode %{
15771 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15772 __ string_compare($str1$$Register, $str2$$Register,
15773 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15774 $tmp1$$Register, $tmp2$$Register,
15775 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15776 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15777 StrIntrinsicNode::LL);
15778 %}
15779 ins_pipe(pipe_class_memory);
15780 %}
15781
15782 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15783 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15784 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15785 pRegGov_P1 pgtmp2, rFlagsReg cr)
15786 %{
15787 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15788 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15789 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15790 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15791
15792 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15793 ins_encode %{
15794 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15795 __ string_compare($str1$$Register, $str2$$Register,
15796 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15797 $tmp1$$Register, $tmp2$$Register,
15798 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15799 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15800 StrIntrinsicNode::LU);
15801 %}
15802 ins_pipe(pipe_class_memory);
15803 %}
15804
15805 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15806 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15807 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15808 pRegGov_P1 pgtmp2, rFlagsReg cr)
15809 %{
15810 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15811 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15812 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15813 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15814
15815 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15816 ins_encode %{
15817 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15818 __ string_compare($str1$$Register, $str2$$Register,
15819 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15820 $tmp1$$Register, $tmp2$$Register,
15821 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15822 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15823 StrIntrinsicNode::UL);
15824 %}
15825 ins_pipe(pipe_class_memory);
15826 %}
15827
15828 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15829 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15830 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15831 pRegGov_P1 pgtmp2, rFlagsReg cr)
15832 %{
15833 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15834 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15835 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15836 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15837
15838 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15839 ins_encode %{
15840 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15841 __ string_compare($str1$$Register, $str2$$Register,
15842 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15843 $tmp1$$Register, $tmp2$$Register,
15844 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15845 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15846 StrIntrinsicNode::UU);
15847 %}
15848 ins_pipe(pipe_class_memory);
15849 %}
15850
15851 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15852 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15853 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15854 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15855 %{
15856 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15857 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15858 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15859 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15860 TEMP vtmp0, TEMP vtmp1, KILL cr);
15861 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15862 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15863
15864 ins_encode %{
15865 __ string_indexof($str1$$Register, $str2$$Register,
15866 $cnt1$$Register, $cnt2$$Register,
15867 $tmp1$$Register, $tmp2$$Register,
15868 $tmp3$$Register, $tmp4$$Register,
15869 $tmp5$$Register, $tmp6$$Register,
15870 -1, $result$$Register, StrIntrinsicNode::UU);
15871 %}
15872 ins_pipe(pipe_class_memory);
15873 %}
15874
15875 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15876 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15877 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15878 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15879 %{
15880 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15881 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15882 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15883 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15884 TEMP vtmp0, TEMP vtmp1, KILL cr);
15885 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15886 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15887
15888 ins_encode %{
15889 __ string_indexof($str1$$Register, $str2$$Register,
15890 $cnt1$$Register, $cnt2$$Register,
15891 $tmp1$$Register, $tmp2$$Register,
15892 $tmp3$$Register, $tmp4$$Register,
15893 $tmp5$$Register, $tmp6$$Register,
15894 -1, $result$$Register, StrIntrinsicNode::LL);
15895 %}
15896 ins_pipe(pipe_class_memory);
15897 %}
15898
15899 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15900 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15901 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15902 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15903 %{
15904 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15905 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15906 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15907 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15908 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15909 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15910 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15911
15912 ins_encode %{
15913 __ string_indexof($str1$$Register, $str2$$Register,
15914 $cnt1$$Register, $cnt2$$Register,
15915 $tmp1$$Register, $tmp2$$Register,
15916 $tmp3$$Register, $tmp4$$Register,
15917 $tmp5$$Register, $tmp6$$Register,
15918 -1, $result$$Register, StrIntrinsicNode::UL);
15919 %}
15920 ins_pipe(pipe_class_memory);
15921 %}
15922
15923 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15924 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15925 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15926 %{
15927 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15928 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15929 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15930 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15931 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15932 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15933
15934 ins_encode %{
15935 int icnt2 = (int)$int_cnt2$$constant;
15936 __ string_indexof($str1$$Register, $str2$$Register,
15937 $cnt1$$Register, zr,
15938 $tmp1$$Register, $tmp2$$Register,
15939 $tmp3$$Register, $tmp4$$Register, zr, zr,
15940 icnt2, $result$$Register, StrIntrinsicNode::UU);
15941 %}
15942 ins_pipe(pipe_class_memory);
15943 %}
15944
15945 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15946 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15947 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15948 %{
15949 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15950 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15951 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15952 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15953 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15954 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15955
15956 ins_encode %{
15957 int icnt2 = (int)$int_cnt2$$constant;
15958 __ string_indexof($str1$$Register, $str2$$Register,
15959 $cnt1$$Register, zr,
15960 $tmp1$$Register, $tmp2$$Register,
15961 $tmp3$$Register, $tmp4$$Register, zr, zr,
15962 icnt2, $result$$Register, StrIntrinsicNode::LL);
15963 %}
15964 ins_pipe(pipe_class_memory);
15965 %}
15966
15967 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15968 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15969 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15970 %{
15971 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15972 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15973 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15974 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15975 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15976 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15977
15978 ins_encode %{
15979 int icnt2 = (int)$int_cnt2$$constant;
15980 __ string_indexof($str1$$Register, $str2$$Register,
15981 $cnt1$$Register, zr,
15982 $tmp1$$Register, $tmp2$$Register,
15983 $tmp3$$Register, $tmp4$$Register, zr, zr,
15984 icnt2, $result$$Register, StrIntrinsicNode::UL);
15985 %}
15986 ins_pipe(pipe_class_memory);
15987 %}
15988
15989 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15990 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15991 iRegINoSp tmp3, rFlagsReg cr)
15992 %{
15993 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15994 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15995 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15996 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15997
15998 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15999
16000 ins_encode %{
16001 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16002 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16003 $tmp3$$Register);
16004 %}
16005 ins_pipe(pipe_class_memory);
16006 %}
16007
16008 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16009 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16010 iRegINoSp tmp3, rFlagsReg cr)
16011 %{
16012 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16013 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16014 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16015 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16016
16017 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16018
16019 ins_encode %{
16020 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16021 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16022 $tmp3$$Register);
16023 %}
16024 ins_pipe(pipe_class_memory);
16025 %}
16026
16027 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16028 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16029 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16030 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16031 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16032 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16033 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16034 ins_encode %{
16035 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16036 $result$$Register, $ztmp1$$FloatRegister,
16037 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16038 $ptmp$$PRegister, true /* isL */);
16039 %}
16040 ins_pipe(pipe_class_memory);
16041 %}
16042
16043 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16044 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16045 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16046 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16047 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16048 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16049 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16050 ins_encode %{
16051 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16052 $result$$Register, $ztmp1$$FloatRegister,
16053 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16054 $ptmp$$PRegister, false /* isL */);
16055 %}
16056 ins_pipe(pipe_class_memory);
16057 %}
16058
16059 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16060 iRegI_R0 result, rFlagsReg cr)
16061 %{
16062 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16063 match(Set result (StrEquals (Binary str1 str2) cnt));
16064 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16065
16066 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16067 ins_encode %{
16068 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16069 __ string_equals($str1$$Register, $str2$$Register,
16070 $result$$Register, $cnt$$Register);
16071 %}
16072 ins_pipe(pipe_class_memory);
16073 %}
16074
16075 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16076 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16077 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16078 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16079 iRegP_R10 tmp, rFlagsReg cr)
16080 %{
16081 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16082 match(Set result (AryEq ary1 ary2));
16083 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16084 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16085 TEMP vtmp6, TEMP vtmp7, KILL cr);
16086
16087 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16088 ins_encode %{
16089 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16090 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16091 $result$$Register, $tmp$$Register, 1);
16092 if (tpc == nullptr) {
16093 ciEnv::current()->record_failure("CodeCache is full");
16094 return;
16095 }
16096 %}
16097 ins_pipe(pipe_class_memory);
16098 %}
16099
16100 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16101 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16102 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16103 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16104 iRegP_R10 tmp, rFlagsReg cr)
16105 %{
16106 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16107 match(Set result (AryEq ary1 ary2));
16108 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16109 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16110 TEMP vtmp6, TEMP vtmp7, KILL cr);
16111
16112 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16113 ins_encode %{
16114 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16115 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16116 $result$$Register, $tmp$$Register, 2);
16117 if (tpc == nullptr) {
16118 ciEnv::current()->record_failure("CodeCache is full");
16119 return;
16120 }
16121 %}
16122 ins_pipe(pipe_class_memory);
16123 %}
16124
16125 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16126 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16127 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16128 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16129 %{
16130 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16131 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16132 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16133
16134 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16135 ins_encode %{
16136 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16137 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16138 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16139 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16140 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16141 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16142 (BasicType)$basic_type$$constant);
16143 if (tpc == nullptr) {
16144 ciEnv::current()->record_failure("CodeCache is full");
16145 return;
16146 }
16147 %}
16148 ins_pipe(pipe_class_memory);
16149 %}
16150
16151 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16152 %{
16153 match(Set result (CountPositives ary1 len));
16154 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16155 format %{ "count positives byte[] $ary1,$len -> $result" %}
16156 ins_encode %{
16157 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16158 if (tpc == nullptr) {
16159 ciEnv::current()->record_failure("CodeCache is full");
16160 return;
16161 }
16162 %}
16163 ins_pipe( pipe_slow );
16164 %}
16165
16166 // fast char[] to byte[] compression
16167 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16168 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16169 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16170 iRegI_R0 result, rFlagsReg cr)
16171 %{
16172 match(Set result (StrCompressedCopy src (Binary dst len)));
16173 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16174 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16175
16176 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16177 ins_encode %{
16178 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16179 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16180 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16181 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16182 %}
16183 ins_pipe(pipe_slow);
16184 %}
16185
16186 // fast byte[] to char[] inflation
16187 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16188 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16189 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16190 %{
16191 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16192 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16193 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16194 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16195
16196 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16197 ins_encode %{
16198 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16199 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16200 $vtmp2$$FloatRegister, $tmp$$Register);
16201 if (tpc == nullptr) {
16202 ciEnv::current()->record_failure("CodeCache is full");
16203 return;
16204 }
16205 %}
16206 ins_pipe(pipe_class_memory);
16207 %}
16208
16209 // encode char[] to byte[] in ISO_8859_1
16210 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16211 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16212 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16213 iRegI_R0 result, rFlagsReg cr)
16214 %{
16215 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16216 match(Set result (EncodeISOArray src (Binary dst len)));
16217 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16218 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16219
16220 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16221 ins_encode %{
16222 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16223 $result$$Register, false,
16224 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16225 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16226 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16227 %}
16228 ins_pipe(pipe_class_memory);
16229 %}
16230
16231 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16232 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16233 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16234 iRegI_R0 result, rFlagsReg cr)
16235 %{
16236 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16237 match(Set result (EncodeISOArray src (Binary dst len)));
16238 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16239 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16240
16241 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16242 ins_encode %{
16243 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16244 $result$$Register, true,
16245 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16246 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16247 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16248 %}
16249 ins_pipe(pipe_class_memory);
16250 %}
16251
16252 //----------------------------- CompressBits/ExpandBits ------------------------
16253
16254 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16255 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16256 match(Set dst (CompressBits src mask));
16257 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16258 format %{ "mov $tsrc, $src\n\t"
16259 "mov $tmask, $mask\n\t"
16260 "bext $tdst, $tsrc, $tmask\n\t"
16261 "mov $dst, $tdst"
16262 %}
16263 ins_encode %{
16264 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16265 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16266 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16267 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16268 %}
16269 ins_pipe(pipe_slow);
16270 %}
16271
16272 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16273 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16274 match(Set dst (CompressBits (LoadI mem) mask));
16275 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16276 format %{ "ldrs $tsrc, $mem\n\t"
16277 "ldrs $tmask, $mask\n\t"
16278 "bext $tdst, $tsrc, $tmask\n\t"
16279 "mov $dst, $tdst"
16280 %}
16281 ins_encode %{
16282 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16283 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16284 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16285 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16286 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16287 %}
16288 ins_pipe(pipe_slow);
16289 %}
16290
16291 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16292 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16293 match(Set dst (CompressBits src mask));
16294 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16295 format %{ "mov $tsrc, $src\n\t"
16296 "mov $tmask, $mask\n\t"
16297 "bext $tdst, $tsrc, $tmask\n\t"
16298 "mov $dst, $tdst"
16299 %}
16300 ins_encode %{
16301 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16302 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16303 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16304 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16305 %}
16306 ins_pipe(pipe_slow);
16307 %}
16308
16309 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16310 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16311 match(Set dst (CompressBits (LoadL mem) mask));
16312 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16313 format %{ "ldrd $tsrc, $mem\n\t"
16314 "ldrd $tmask, $mask\n\t"
16315 "bext $tdst, $tsrc, $tmask\n\t"
16316 "mov $dst, $tdst"
16317 %}
16318 ins_encode %{
16319 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16320 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16321 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16322 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16323 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16324 %}
16325 ins_pipe(pipe_slow);
16326 %}
16327
16328 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16329 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16330 match(Set dst (ExpandBits src mask));
16331 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16332 format %{ "mov $tsrc, $src\n\t"
16333 "mov $tmask, $mask\n\t"
16334 "bdep $tdst, $tsrc, $tmask\n\t"
16335 "mov $dst, $tdst"
16336 %}
16337 ins_encode %{
16338 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16339 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16340 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16341 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16342 %}
16343 ins_pipe(pipe_slow);
16344 %}
16345
16346 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16347 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16348 match(Set dst (ExpandBits (LoadI mem) mask));
16349 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16350 format %{ "ldrs $tsrc, $mem\n\t"
16351 "ldrs $tmask, $mask\n\t"
16352 "bdep $tdst, $tsrc, $tmask\n\t"
16353 "mov $dst, $tdst"
16354 %}
16355 ins_encode %{
16356 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16357 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16358 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16359 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16360 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16361 %}
16362 ins_pipe(pipe_slow);
16363 %}
16364
16365 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16366 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16367 match(Set dst (ExpandBits src mask));
16368 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16369 format %{ "mov $tsrc, $src\n\t"
16370 "mov $tmask, $mask\n\t"
16371 "bdep $tdst, $tsrc, $tmask\n\t"
16372 "mov $dst, $tdst"
16373 %}
16374 ins_encode %{
16375 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16376 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16377 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16378 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16379 %}
16380 ins_pipe(pipe_slow);
16381 %}
16382
16383
16384 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16385 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16386 match(Set dst (ExpandBits (LoadL mem) mask));
16387 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16388 format %{ "ldrd $tsrc, $mem\n\t"
16389 "ldrd $tmask, $mask\n\t"
16390 "bdep $tdst, $tsrc, $tmask\n\t"
16391 "mov $dst, $tdst"
16392 %}
16393 ins_encode %{
16394 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16395 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16396 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16397 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16398 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16399 %}
16400 ins_pipe(pipe_slow);
16401 %}
16402
16403 //----------------------------- Reinterpret ----------------------------------
16404 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16405 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16406 match(Set dst (ReinterpretHF2S src));
16407 format %{ "reinterpretHF2S $dst, $src" %}
16408 ins_encode %{
16409 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16410 %}
16411 ins_pipe(pipe_slow);
16412 %}
16413
16414 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16415 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16416 match(Set dst (ReinterpretS2HF src));
16417 format %{ "reinterpretS2HF $dst, $src" %}
16418 ins_encode %{
16419 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16420 %}
16421 ins_pipe(pipe_slow);
16422 %}
16423
16424 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16425 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16426 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16427 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16428 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16429 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16430 // can be omitted in this pattern, resulting in -
16431 // fcvt $dst, $src // Convert float to half-precision float
16432 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16433 %{
16434 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16435 format %{ "convF2HFAndS2HF $dst, $src" %}
16436 ins_encode %{
16437 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16438 %}
16439 ins_pipe(pipe_slow);
16440 %}
16441
16442 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16443 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16444 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16445 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16446 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16447 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16448 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16449 // resulting in -
16450 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16451 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16452 %{
16453 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16454 format %{ "convHF2SAndHF2F $dst, $src" %}
16455 ins_encode %{
16456 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16457 %}
16458 ins_pipe(pipe_slow);
16459 %}
16460
16461 // ============================================================================
16462 // This name is KNOWN by the ADLC and cannot be changed.
16463 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16464 // for this guy.
16465 instruct tlsLoadP(thread_RegP dst)
16466 %{
16467 match(Set dst (ThreadLocal));
16468
16469 ins_cost(0);
16470
16471 format %{ " -- \t// $dst=Thread::current(), empty" %}
16472
16473 size(0);
16474
16475 ins_encode( /*empty*/ );
16476
16477 ins_pipe(pipe_class_empty);
16478 %}
16479
16480 //----------PEEPHOLE RULES-----------------------------------------------------
16481 // These must follow all instruction definitions as they use the names
16482 // defined in the instructions definitions.
16483 //
16484 // peepmatch ( root_instr_name [preceding_instruction]* );
16485 //
16486 // peepconstraint %{
16487 // (instruction_number.operand_name relational_op instruction_number.operand_name
16488 // [, ...] );
16489 // // instruction numbers are zero-based using left to right order in peepmatch
16490 //
16491 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16492 // // provide an instruction_number.operand_name for each operand that appears
16493 // // in the replacement instruction's match rule
16494 //
16495 // ---------VM FLAGS---------------------------------------------------------
16496 //
16497 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16498 //
16499 // Each peephole rule is given an identifying number starting with zero and
16500 // increasing by one in the order seen by the parser. An individual peephole
16501 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16502 // on the command-line.
16503 //
16504 // ---------CURRENT LIMITATIONS----------------------------------------------
16505 //
16506 // Only match adjacent instructions in same basic block
16507 // Only equality constraints
16508 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16509 // Only one replacement instruction
16510 //
16511 // ---------EXAMPLE----------------------------------------------------------
16512 //
16513 // // pertinent parts of existing instructions in architecture description
16514 // instruct movI(iRegINoSp dst, iRegI src)
16515 // %{
16516 // match(Set dst (CopyI src));
16517 // %}
16518 //
16519 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16520 // %{
16521 // match(Set dst (AddI dst src));
16522 // effect(KILL cr);
16523 // %}
16524 //
16525 // // Change (inc mov) to lea
16526 // peephole %{
16527 // // increment preceded by register-register move
16528 // peepmatch ( incI_iReg movI );
16529 // // require that the destination register of the increment
16530 // // match the destination register of the move
16531 // peepconstraint ( 0.dst == 1.dst );
16532 // // construct a replacement instruction that sets
16533 // // the destination to ( move's source register + one )
16534 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16535 // %}
16536 //
16537
16538 // Implementation no longer uses movX instructions since
16539 // machine-independent system no longer uses CopyX nodes.
16540 //
16541 // peephole
16542 // %{
16543 // peepmatch (incI_iReg movI);
16544 // peepconstraint (0.dst == 1.dst);
16545 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16546 // %}
16547
16548 // peephole
16549 // %{
16550 // peepmatch (decI_iReg movI);
16551 // peepconstraint (0.dst == 1.dst);
16552 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16553 // %}
16554
16555 // peephole
16556 // %{
16557 // peepmatch (addI_iReg_imm movI);
16558 // peepconstraint (0.dst == 1.dst);
16559 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16560 // %}
16561
16562 // peephole
16563 // %{
16564 // peepmatch (incL_iReg movL);
16565 // peepconstraint (0.dst == 1.dst);
16566 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16567 // %}
16568
16569 // peephole
16570 // %{
16571 // peepmatch (decL_iReg movL);
16572 // peepconstraint (0.dst == 1.dst);
16573 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16574 // %}
16575
16576 // peephole
16577 // %{
16578 // peepmatch (addL_iReg_imm movL);
16579 // peepconstraint (0.dst == 1.dst);
16580 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16581 // %}
16582
16583 // peephole
16584 // %{
16585 // peepmatch (addP_iReg_imm movP);
16586 // peepconstraint (0.dst == 1.dst);
16587 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16588 // %}
16589
16590 // // Change load of spilled value to only a spill
16591 // instruct storeI(memory mem, iRegI src)
16592 // %{
16593 // match(Set mem (StoreI mem src));
16594 // %}
16595 //
16596 // instruct loadI(iRegINoSp dst, memory mem)
16597 // %{
16598 // match(Set dst (LoadI mem));
16599 // %}
16600 //
16601
16602 //----------SMARTSPILL RULES---------------------------------------------------
16603 // These must follow all instruction definitions as they use the names
16604 // defined in the instructions definitions.
16605
16606 // Local Variables:
16607 // mode: c++
16608 // End: