1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return MacroAssembler::max_trampoline_stub_size();
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 5; // metadata; call dest; trampoline address; trampoline destination; trampoline_owner_metadata
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else {
1694 return 6 * NativeInstruction::instruction_size;
1695 }
1696 }
1697
1698 //=============================================================================
1699
1700 #ifndef PRODUCT
1701 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1702 st->print("BREAKPOINT");
1703 }
1704 #endif
1705
1706 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1707 __ brk(0);
1708 }
1709
1710 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1711 return MachNode::size(ra_);
1712 }
1713
1714 //=============================================================================
1715
1716 #ifndef PRODUCT
1717 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1718 st->print("nop \t# %d bytes pad for loops and calls", _count);
1719 }
1720 #endif
1721
1722 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1723 for (int i = 0; i < _count; i++) {
1724 __ nop();
1725 }
1726 }
1727
1728 uint MachNopNode::size(PhaseRegAlloc*) const {
1729 return _count * NativeInstruction::instruction_size;
1730 }
1731
1732 //=============================================================================
1733 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1734
1735 int ConstantTable::calculate_table_base_offset() const {
1736 return 0; // absolute addressing, no offset
1737 }
1738
1739 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1740 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1741 ShouldNotReachHere();
1742 }
1743
1744 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1745 // Empty encoding
1746 }
1747
1748 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1749 return 0;
1750 }
1751
1752 #ifndef PRODUCT
1753 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1754 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1755 }
1756 #endif
1757
1758 #ifndef PRODUCT
1759 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1760 Compile* C = ra_->C;
1761
1762 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1763
1764 if (C->output()->need_stack_bang(framesize))
1765 st->print("# stack bang size=%d\n\t", framesize);
1766
1767 if (VM_Version::use_rop_protection()) {
1768 st->print("ldr zr, [lr]\n\t");
1769 st->print("paciaz\n\t");
1770 }
1771 if (framesize < ((1 << 9) + 2 * wordSize)) {
1772 st->print("sub sp, sp, #%d\n\t", framesize);
1773 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1774 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1775 } else {
1776 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1777 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1778 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1779 st->print("sub sp, sp, rscratch1");
1780 }
1781 if (C->stub_function() == nullptr) {
1782 st->print("\n\t");
1783 st->print("ldr rscratch1, [guard]\n\t");
1784 st->print("dmb ishld\n\t");
1785 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1786 st->print("cmp rscratch1, rscratch2\n\t");
1787 st->print("b.eq skip");
1788 st->print("\n\t");
1789 st->print("blr #nmethod_entry_barrier_stub\n\t");
1790 st->print("b skip\n\t");
1791 st->print("guard: int\n\t");
1792 st->print("\n\t");
1793 st->print("skip:\n\t");
1794 }
1795 }
1796 #endif
1797
1798 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1799 Compile* C = ra_->C;
1800
1801 // n.b. frame size includes space for return pc and rfp
1802 const int framesize = C->output()->frame_size_in_bytes();
1803
1804 if (C->clinit_barrier_on_entry()) {
1805 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1806
1807 Label L_skip_barrier;
1808
1809 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1810 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1811 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1812 __ bind(L_skip_barrier);
1813 }
1814
1815 if (C->max_vector_size() > 0) {
1816 __ reinitialize_ptrue();
1817 }
1818
1819 int bangsize = C->output()->bang_size_in_bytes();
1820 if (C->output()->need_stack_bang(bangsize))
1821 __ generate_stack_overflow_check(bangsize);
1822
1823 __ build_frame(framesize);
1824
1825 if (C->stub_function() == nullptr) {
1826 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1827 // Dummy labels for just measuring the code size
1828 Label dummy_slow_path;
1829 Label dummy_continuation;
1830 Label dummy_guard;
1831 Label* slow_path = &dummy_slow_path;
1832 Label* continuation = &dummy_continuation;
1833 Label* guard = &dummy_guard;
1834 if (!Compile::current()->output()->in_scratch_emit_size()) {
1835 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1836 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1837 Compile::current()->output()->add_stub(stub);
1838 slow_path = &stub->entry();
1839 continuation = &stub->continuation();
1840 guard = &stub->guard();
1841 }
1842 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1843 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1844 }
1845
1846 if (VerifyStackAtCalls) {
1847 Unimplemented();
1848 }
1849
1850 C->output()->set_frame_complete(__ offset());
1851
1852 if (C->has_mach_constant_base_node()) {
1853 // NOTE: We set the table base offset here because users might be
1854 // emitted before MachConstantBaseNode.
1855 ConstantTable& constant_table = C->output()->constant_table();
1856 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1857 }
1858 }
1859
1860 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1861 {
1862 return MachNode::size(ra_); // too many variables; just compute it
1863 // the hard way
1864 }
1865
1866 int MachPrologNode::reloc() const
1867 {
1868 return 0;
1869 }
1870
1871 //=============================================================================
1872
1873 #ifndef PRODUCT
1874 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 st->print("# pop frame %d\n\t",framesize);
1879
1880 if (framesize == 0) {
1881 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1882 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1883 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1884 st->print("add sp, sp, #%d\n\t", framesize);
1885 } else {
1886 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1887 st->print("add sp, sp, rscratch1\n\t");
1888 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1889 }
1890 if (VM_Version::use_rop_protection()) {
1891 st->print("autiaz\n\t");
1892 st->print("ldr zr, [lr]\n\t");
1893 }
1894
1895 if (do_polling() && C->is_method_compilation()) {
1896 st->print("# test polling word\n\t");
1897 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1898 st->print("cmp sp, rscratch1\n\t");
1899 st->print("bhi #slow_path");
1900 }
1901 }
1902 #endif
1903
1904 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1905 Compile* C = ra_->C;
1906 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1907
1908 __ remove_frame(framesize);
1909
1910 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1911 __ reserved_stack_check();
1912 }
1913
1914 if (do_polling() && C->is_method_compilation()) {
1915 Label dummy_label;
1916 Label* code_stub = &dummy_label;
1917 if (!C->output()->in_scratch_emit_size()) {
1918 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1919 C->output()->add_stub(stub);
1920 code_stub = &stub->entry();
1921 }
1922 __ relocate(relocInfo::poll_return_type);
1923 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1924 }
1925 }
1926
1927 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1928 // Variable size. Determine dynamically.
1929 return MachNode::size(ra_);
1930 }
1931
1932 int MachEpilogNode::reloc() const {
1933 // Return number of relocatable values contained in this instruction.
1934 return 1; // 1 for polling page.
1935 }
1936
1937 const Pipeline * MachEpilogNode::pipeline() const {
1938 return MachNode::pipeline_class();
1939 }
1940
1941 //=============================================================================
1942
1943 static enum RC rc_class(OptoReg::Name reg) {
1944
1945 if (reg == OptoReg::Bad) {
1946 return rc_bad;
1947 }
1948
1949 // we have 32 int registers * 2 halves
1950 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1951
1952 if (reg < slots_of_int_registers) {
1953 return rc_int;
1954 }
1955
1956 // we have 32 float register * 8 halves
1957 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1958 if (reg < slots_of_int_registers + slots_of_float_registers) {
1959 return rc_float;
1960 }
1961
1962 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1963 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1964 return rc_predicate;
1965 }
1966
1967 // Between predicate regs & stack is the flags.
1968 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1969
1970 return rc_stack;
1971 }
1972
1973 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1974 Compile* C = ra_->C;
1975
1976 // Get registers to move.
1977 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1978 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1979 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1980 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1981
1982 enum RC src_hi_rc = rc_class(src_hi);
1983 enum RC src_lo_rc = rc_class(src_lo);
1984 enum RC dst_hi_rc = rc_class(dst_hi);
1985 enum RC dst_lo_rc = rc_class(dst_lo);
1986
1987 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1988
1989 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1990 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1991 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1992 "expected aligned-adjacent pairs");
1993 }
1994
1995 if (src_lo == dst_lo && src_hi == dst_hi) {
1996 return 0; // Self copy, no move.
1997 }
1998
1999 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2000 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2001 int src_offset = ra_->reg2offset(src_lo);
2002 int dst_offset = ra_->reg2offset(dst_lo);
2003
2004 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2005 uint ireg = ideal_reg();
2006 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
2007 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
2008 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
2009 if (ireg == Op_VecA && masm) {
2010 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2011 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2012 // stack->stack
2013 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2016 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2017 sve_vector_reg_size_in_bytes);
2018 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2019 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2020 sve_vector_reg_size_in_bytes);
2021 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2022 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2023 as_FloatRegister(Matcher::_regEncode[src_lo]),
2024 as_FloatRegister(Matcher::_regEncode[src_lo]));
2025 } else {
2026 ShouldNotReachHere();
2027 }
2028 } else if (masm) {
2029 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2030 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2031 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2032 // stack->stack
2033 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2034 if (ireg == Op_VecD) {
2035 __ unspill(rscratch1, true, src_offset);
2036 __ spill(rscratch1, true, dst_offset);
2037 } else {
2038 __ spill_copy128(src_offset, dst_offset);
2039 }
2040 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2041 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2042 ireg == Op_VecD ? __ T8B : __ T16B,
2043 as_FloatRegister(Matcher::_regEncode[src_lo]));
2044 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2045 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2046 ireg == Op_VecD ? __ D : __ Q,
2047 ra_->reg2offset(dst_lo));
2048 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2049 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2050 ireg == Op_VecD ? __ D : __ Q,
2051 ra_->reg2offset(src_lo));
2052 } else {
2053 ShouldNotReachHere();
2054 }
2055 }
2056 } else if (masm) {
2057 switch (src_lo_rc) {
2058 case rc_int:
2059 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2060 if (is64) {
2061 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2062 as_Register(Matcher::_regEncode[src_lo]));
2063 } else {
2064 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2065 as_Register(Matcher::_regEncode[src_lo]));
2066 }
2067 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2068 if (is64) {
2069 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2070 as_Register(Matcher::_regEncode[src_lo]));
2071 } else {
2072 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2073 as_Register(Matcher::_regEncode[src_lo]));
2074 }
2075 } else { // gpr --> stack spill
2076 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2077 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2078 }
2079 break;
2080 case rc_float:
2081 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2082 if (is64) {
2083 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2084 as_FloatRegister(Matcher::_regEncode[src_lo]));
2085 } else {
2086 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2087 as_FloatRegister(Matcher::_regEncode[src_lo]));
2088 }
2089 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2090 if (is64) {
2091 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2092 as_FloatRegister(Matcher::_regEncode[src_lo]));
2093 } else {
2094 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2095 as_FloatRegister(Matcher::_regEncode[src_lo]));
2096 }
2097 } else { // fpr --> stack spill
2098 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2099 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2100 is64 ? __ D : __ S, dst_offset);
2101 }
2102 break;
2103 case rc_stack:
2104 if (dst_lo_rc == rc_int) { // stack --> gpr load
2105 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2106 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2107 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2108 is64 ? __ D : __ S, src_offset);
2109 } else if (dst_lo_rc == rc_predicate) {
2110 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2111 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2112 } else { // stack --> stack copy
2113 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2114 if (ideal_reg() == Op_RegVectMask) {
2115 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2116 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2117 } else {
2118 __ unspill(rscratch1, is64, src_offset);
2119 __ spill(rscratch1, is64, dst_offset);
2120 }
2121 }
2122 break;
2123 case rc_predicate:
2124 if (dst_lo_rc == rc_predicate) {
2125 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2126 } else if (dst_lo_rc == rc_stack) {
2127 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2128 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2129 } else {
2130 assert(false, "bad src and dst rc_class combination.");
2131 ShouldNotReachHere();
2132 }
2133 break;
2134 default:
2135 assert(false, "bad rc_class for spill");
2136 ShouldNotReachHere();
2137 }
2138 }
2139
2140 if (st) {
2141 st->print("spill ");
2142 if (src_lo_rc == rc_stack) {
2143 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2144 } else {
2145 st->print("%s -> ", Matcher::regName[src_lo]);
2146 }
2147 if (dst_lo_rc == rc_stack) {
2148 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2149 } else {
2150 st->print("%s", Matcher::regName[dst_lo]);
2151 }
2152 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2153 int vsize = 0;
2154 switch (ideal_reg()) {
2155 case Op_VecD:
2156 vsize = 64;
2157 break;
2158 case Op_VecX:
2159 vsize = 128;
2160 break;
2161 case Op_VecA:
2162 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2163 break;
2164 default:
2165 assert(false, "bad register type for spill");
2166 ShouldNotReachHere();
2167 }
2168 st->print("\t# vector spill size = %d", vsize);
2169 } else if (ideal_reg() == Op_RegVectMask) {
2170 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2171 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2172 st->print("\t# predicate spill size = %d", vsize);
2173 } else {
2174 st->print("\t# spill size = %d", is64 ? 64 : 32);
2175 }
2176 }
2177
2178 return 0;
2179
2180 }
2181
2182 #ifndef PRODUCT
2183 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2184 if (!ra_)
2185 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2186 else
2187 implementation(nullptr, ra_, false, st);
2188 }
2189 #endif
2190
2191 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2192 implementation(masm, ra_, false, nullptr);
2193 }
2194
2195 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2196 return MachNode::size(ra_);
2197 }
2198
2199 //=============================================================================
2200
2201 #ifndef PRODUCT
2202 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2203 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2204 int reg = ra_->get_reg_first(this);
2205 st->print("add %s, rsp, #%d]\t# box lock",
2206 Matcher::regName[reg], offset);
2207 }
2208 #endif
2209
2210 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2211 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2212 int reg = ra_->get_encode(this);
2213
2214 // This add will handle any 24-bit signed offset. 24 bits allows an
2215 // 8 megabyte stack frame.
2216 __ add(as_Register(reg), sp, offset);
2217 }
2218
2219 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2220 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2221 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2222
2223 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2224 return NativeInstruction::instruction_size;
2225 } else {
2226 return 2 * NativeInstruction::instruction_size;
2227 }
2228 }
2229
2230 //=============================================================================
2231
2232 #ifndef PRODUCT
2233 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2234 {
2235 st->print_cr("# MachUEPNode");
2236 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2237 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2238 st->print_cr("\tcmpw rscratch1, r10");
2239 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2240 }
2241 #endif
2242
2243 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2244 {
2245 __ ic_check(InteriorEntryAlignment);
2246 }
2247
2248 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2249 {
2250 return MachNode::size(ra_);
2251 }
2252
2253 // REQUIRED EMIT CODE
2254
2255 //=============================================================================
2256
2257 // Emit deopt handler code.
2258 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2259 {
2260 // Note that the code buffer's insts_mark is always relative to insts.
2261 // That's why we must use the macroassembler to generate a handler.
2262 address base = __ start_a_stub(size_deopt_handler());
2263 if (base == nullptr) {
2264 ciEnv::current()->record_failure("CodeCache is full");
2265 return 0; // CodeBuffer::expand failed
2266 }
2267
2268 int offset = __ offset();
2269 Label start;
2270 __ bind(start);
2271 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2272
2273 int entry_offset = __ offset();
2274 __ b(start);
2275
2276 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2277 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2278 "out of bounds read in post-call NOP check");
2279 __ end_a_stub();
2280 return entry_offset;
2281 }
2282
2283 // REQUIRED MATCHER CODE
2284
2285 //=============================================================================
2286
2287 bool Matcher::match_rule_supported(int opcode) {
2288 if (!has_match_rule(opcode))
2289 return false;
2290
2291 switch (opcode) {
2292 case Op_OnSpinWait:
2293 return VM_Version::supports_on_spin_wait();
2294 case Op_CacheWB:
2295 case Op_CacheWBPreSync:
2296 case Op_CacheWBPostSync:
2297 if (!VM_Version::supports_data_cache_line_flush()) {
2298 return false;
2299 }
2300 break;
2301 case Op_ExpandBits:
2302 case Op_CompressBits:
2303 if (!VM_Version::supports_svebitperm()) {
2304 return false;
2305 }
2306 break;
2307 case Op_FmaF:
2308 case Op_FmaD:
2309 case Op_FmaVF:
2310 case Op_FmaVD:
2311 if (!UseFMA) {
2312 return false;
2313 }
2314 break;
2315 case Op_FmaHF:
2316 // UseFMA flag also needs to be checked along with FEAT_FP16
2317 if (!UseFMA || !is_feat_fp16_supported()) {
2318 return false;
2319 }
2320 break;
2321 case Op_AddHF:
2322 case Op_SubHF:
2323 case Op_MulHF:
2324 case Op_DivHF:
2325 case Op_MinHF:
2326 case Op_MaxHF:
2327 case Op_SqrtHF:
2328 // Half-precision floating point scalar operations require FEAT_FP16
2329 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2330 // features are supported.
2331 if (!is_feat_fp16_supported()) {
2332 return false;
2333 }
2334 break;
2335 }
2336
2337 return true; // Per default match rules are supported.
2338 }
2339
2340 const RegMask* Matcher::predicate_reg_mask(void) {
2341 return &_PR_REG_mask;
2342 }
2343
2344 bool Matcher::supports_vector_calling_convention(void) {
2345 return EnableVectorSupport;
2346 }
2347
2348 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2349 assert(EnableVectorSupport, "sanity");
2350 int lo = V0_num;
2351 int hi = V0_H_num;
2352 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2353 hi = V0_K_num;
2354 }
2355 return OptoRegPair(hi, lo);
2356 }
2357
2358 // Is this branch offset short enough that a short branch can be used?
2359 //
2360 // NOTE: If the platform does not provide any short branch variants, then
2361 // this method should return false for offset 0.
2362 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2363 // The passed offset is relative to address of the branch.
2364
2365 return (-32768 <= offset && offset < 32768);
2366 }
2367
2368 // Vector width in bytes.
2369 int Matcher::vector_width_in_bytes(BasicType bt) {
2370 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2371 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2372 // Minimum 2 values in vector
2373 if (size < 2*type2aelembytes(bt)) size = 0;
2374 // But never < 4
2375 if (size < 4) size = 0;
2376 return size;
2377 }
2378
2379 // Limits on vector size (number of elements) loaded into vector.
2380 int Matcher::max_vector_size(const BasicType bt) {
2381 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2382 }
2383
2384 int Matcher::min_vector_size(const BasicType bt) {
2385 // Usually, the shortest vector length supported by AArch64 ISA and
2386 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2387 // vectors in a few special cases.
2388 int size;
2389 switch(bt) {
2390 case T_BOOLEAN:
2391 // Load/store a vector mask with only 2 elements for vector types
2392 // such as "2I/2F/2L/2D".
2393 size = 2;
2394 break;
2395 case T_BYTE:
2396 // Generate a "4B" vector, to support vector cast between "8B/16B"
2397 // and "4S/4I/4L/4F/4D".
2398 size = 4;
2399 break;
2400 case T_SHORT:
2401 // Generate a "2S" vector, to support vector cast between "4S/8S"
2402 // and "2I/2L/2F/2D".
2403 size = 2;
2404 break;
2405 default:
2406 // Limit the min vector length to 64-bit.
2407 size = 8 / type2aelembytes(bt);
2408 // The number of elements in a vector should be at least 2.
2409 size = MAX2(size, 2);
2410 }
2411
2412 int max_size = max_vector_size(bt);
2413 return MIN2(size, max_size);
2414 }
2415
2416 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2417 return Matcher::max_vector_size(bt);
2418 }
2419
2420 // Actual max scalable vector register length.
2421 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2422 return Matcher::max_vector_size(bt);
2423 }
2424
2425 // Vector ideal reg.
2426 uint Matcher::vector_ideal_reg(int len) {
2427 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2428 return Op_VecA;
2429 }
2430 switch(len) {
2431 // For 16-bit/32-bit mask vector, reuse VecD.
2432 case 2:
2433 case 4:
2434 case 8: return Op_VecD;
2435 case 16: return Op_VecX;
2436 }
2437 ShouldNotReachHere();
2438 return 0;
2439 }
2440
2441 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2442 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2443 switch (ideal_reg) {
2444 case Op_VecA: return new vecAOper();
2445 case Op_VecD: return new vecDOper();
2446 case Op_VecX: return new vecXOper();
2447 }
2448 ShouldNotReachHere();
2449 return nullptr;
2450 }
2451
2452 bool Matcher::is_reg2reg_move(MachNode* m) {
2453 return false;
2454 }
2455
2456 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2457 return false;
2458 }
2459
2460 bool Matcher::is_generic_vector(MachOper* opnd) {
2461 return opnd->opcode() == VREG;
2462 }
2463
2464 #ifdef ASSERT
2465 // Return whether or not this register is ever used as an argument.
2466 bool Matcher::can_be_java_arg(int reg)
2467 {
2468 return
2469 reg == R0_num || reg == R0_H_num ||
2470 reg == R1_num || reg == R1_H_num ||
2471 reg == R2_num || reg == R2_H_num ||
2472 reg == R3_num || reg == R3_H_num ||
2473 reg == R4_num || reg == R4_H_num ||
2474 reg == R5_num || reg == R5_H_num ||
2475 reg == R6_num || reg == R6_H_num ||
2476 reg == R7_num || reg == R7_H_num ||
2477 reg == V0_num || reg == V0_H_num ||
2478 reg == V1_num || reg == V1_H_num ||
2479 reg == V2_num || reg == V2_H_num ||
2480 reg == V3_num || reg == V3_H_num ||
2481 reg == V4_num || reg == V4_H_num ||
2482 reg == V5_num || reg == V5_H_num ||
2483 reg == V6_num || reg == V6_H_num ||
2484 reg == V7_num || reg == V7_H_num;
2485 }
2486 #endif
2487
2488 uint Matcher::int_pressure_limit()
2489 {
2490 // JDK-8183543: When taking the number of available registers as int
2491 // register pressure threshold, the jtreg test:
2492 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2493 // failed due to C2 compilation failure with
2494 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2495 //
2496 // A derived pointer is live at CallNode and then is flagged by RA
2497 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2498 // derived pointers and lastly fail to spill after reaching maximum
2499 // number of iterations. Lowering the default pressure threshold to
2500 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2501 // a high register pressure area of the code so that split_DEF can
2502 // generate DefinitionSpillCopy for the derived pointer.
2503 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2504 if (!PreserveFramePointer) {
2505 // When PreserveFramePointer is off, frame pointer is allocatable,
2506 // but different from other SOC registers, it is excluded from
2507 // fatproj's mask because its save type is No-Save. Decrease 1 to
2508 // ensure high pressure at fatproj when PreserveFramePointer is off.
2509 // See check_pressure_at_fatproj().
2510 default_int_pressure_threshold--;
2511 }
2512 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2513 }
2514
2515 uint Matcher::float_pressure_limit()
2516 {
2517 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2518 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2519 }
2520
2521 const RegMask& Matcher::divI_proj_mask() {
2522 ShouldNotReachHere();
2523 return RegMask::EMPTY;
2524 }
2525
2526 // Register for MODI projection of divmodI.
2527 const RegMask& Matcher::modI_proj_mask() {
2528 ShouldNotReachHere();
2529 return RegMask::EMPTY;
2530 }
2531
2532 // Register for DIVL projection of divmodL.
2533 const RegMask& Matcher::divL_proj_mask() {
2534 ShouldNotReachHere();
2535 return RegMask::EMPTY;
2536 }
2537
2538 // Register for MODL projection of divmodL.
2539 const RegMask& Matcher::modL_proj_mask() {
2540 ShouldNotReachHere();
2541 return RegMask::EMPTY;
2542 }
2543
2544 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2545 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2546 Node* u = addp->fast_out(i);
2547 if (u->is_LoadStore()) {
2548 // On AArch64, LoadStoreNodes (i.e. compare and swap
2549 // instructions) only take register indirect as an operand, so
2550 // any attempt to use an AddPNode as an input to a LoadStoreNode
2551 // must fail.
2552 return false;
2553 }
2554 if (u->is_Mem()) {
2555 int opsize = u->as_Mem()->memory_size();
2556 assert(opsize > 0, "unexpected memory operand size");
2557 if (u->as_Mem()->memory_size() != (1<<shift)) {
2558 return false;
2559 }
2560 }
2561 }
2562 return true;
2563 }
2564
2565 // Convert BoolTest condition to Assembler condition.
2566 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2567 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2568 Assembler::Condition result;
2569 switch(cond) {
2570 case BoolTest::eq:
2571 result = Assembler::EQ; break;
2572 case BoolTest::ne:
2573 result = Assembler::NE; break;
2574 case BoolTest::le:
2575 result = Assembler::LE; break;
2576 case BoolTest::ge:
2577 result = Assembler::GE; break;
2578 case BoolTest::lt:
2579 result = Assembler::LT; break;
2580 case BoolTest::gt:
2581 result = Assembler::GT; break;
2582 case BoolTest::ule:
2583 result = Assembler::LS; break;
2584 case BoolTest::uge:
2585 result = Assembler::HS; break;
2586 case BoolTest::ult:
2587 result = Assembler::LO; break;
2588 case BoolTest::ugt:
2589 result = Assembler::HI; break;
2590 case BoolTest::overflow:
2591 result = Assembler::VS; break;
2592 case BoolTest::no_overflow:
2593 result = Assembler::VC; break;
2594 default:
2595 ShouldNotReachHere();
2596 return Assembler::Condition(-1);
2597 }
2598
2599 // Check conversion
2600 if (cond & BoolTest::unsigned_compare) {
2601 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2602 } else {
2603 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2604 }
2605
2606 return result;
2607 }
2608
2609 // Binary src (Replicate con)
2610 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2611 if (n == nullptr || m == nullptr) {
2612 return false;
2613 }
2614
2615 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2616 return false;
2617 }
2618
2619 Node* imm_node = m->in(1);
2620 if (!imm_node->is_Con()) {
2621 return false;
2622 }
2623
2624 const Type* t = imm_node->bottom_type();
2625 if (!(t->isa_int() || t->isa_long())) {
2626 return false;
2627 }
2628
2629 switch (n->Opcode()) {
2630 case Op_AndV:
2631 case Op_OrV:
2632 case Op_XorV: {
2633 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2634 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2635 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2636 }
2637 case Op_AddVB:
2638 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2639 case Op_AddVS:
2640 case Op_AddVI:
2641 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2642 case Op_AddVL:
2643 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2644 default:
2645 return false;
2646 }
2647 }
2648
2649 // (XorV src (Replicate m1))
2650 // (XorVMask src (MaskAll m1))
2651 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2652 if (n != nullptr && m != nullptr) {
2653 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2654 VectorNode::is_all_ones_vector(m);
2655 }
2656 return false;
2657 }
2658
2659 // Should the matcher clone input 'm' of node 'n'?
2660 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2661 if (is_vshift_con_pattern(n, m) ||
2662 is_vector_bitwise_not_pattern(n, m) ||
2663 is_valid_sve_arith_imm_pattern(n, m) ||
2664 is_encode_and_store_pattern(n, m)) {
2665 mstack.push(m, Visit);
2666 return true;
2667 }
2668 return false;
2669 }
2670
2671 // Should the Matcher clone shifts on addressing modes, expecting them
2672 // to be subsumed into complex addressing expressions or compute them
2673 // into registers?
2674 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2675
2676 // Loads and stores with indirect memory input (e.g., volatile loads and
2677 // stores) do not subsume the input into complex addressing expressions. If
2678 // the addressing expression is input to at least one such load or store, do
2679 // not clone the addressing expression. Query needs_acquiring_load and
2680 // needs_releasing_store as a proxy for indirect memory input, as it is not
2681 // possible to directly query for indirect memory input at this stage.
2682 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2683 Node* n = m->fast_out(i);
2684 if (n->is_Load() && needs_acquiring_load(n)) {
2685 return false;
2686 }
2687 if (n->is_Store() && needs_releasing_store(n)) {
2688 return false;
2689 }
2690 }
2691
2692 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2693 return true;
2694 }
2695
2696 Node *off = m->in(AddPNode::Offset);
2697 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2698 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2699 // Are there other uses besides address expressions?
2700 !is_visited(off)) {
2701 address_visited.set(off->_idx); // Flag as address_visited
2702 mstack.push(off->in(2), Visit);
2703 Node *conv = off->in(1);
2704 if (conv->Opcode() == Op_ConvI2L &&
2705 // Are there other uses besides address expressions?
2706 !is_visited(conv)) {
2707 address_visited.set(conv->_idx); // Flag as address_visited
2708 mstack.push(conv->in(1), Pre_Visit);
2709 } else {
2710 mstack.push(conv, Pre_Visit);
2711 }
2712 address_visited.test_set(m->_idx); // Flag as address_visited
2713 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2714 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2715 return true;
2716 } else if (off->Opcode() == Op_ConvI2L &&
2717 // Are there other uses besides address expressions?
2718 !is_visited(off)) {
2719 address_visited.test_set(m->_idx); // Flag as address_visited
2720 address_visited.set(off->_idx); // Flag as address_visited
2721 mstack.push(off->in(1), Pre_Visit);
2722 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2723 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2724 return true;
2725 }
2726 return false;
2727 }
2728
2729 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2730 { \
2731 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2732 guarantee(DISP == 0, "mode not permitted for volatile"); \
2733 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2734 __ INSN(REG, as_Register(BASE)); \
2735 }
2736
2737
2738 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2739 {
2740 Address::extend scale;
2741
2742 // Hooboy, this is fugly. We need a way to communicate to the
2743 // encoder that the index needs to be sign extended, so we have to
2744 // enumerate all the cases.
2745 switch (opcode) {
2746 case INDINDEXSCALEDI2L:
2747 case INDINDEXSCALEDI2LN:
2748 case INDINDEXI2L:
2749 case INDINDEXI2LN:
2750 scale = Address::sxtw(size);
2751 break;
2752 default:
2753 scale = Address::lsl(size);
2754 }
2755
2756 if (index == -1) {
2757 return Address(base, disp);
2758 } else {
2759 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2760 return Address(base, as_Register(index), scale);
2761 }
2762 }
2763
2764
2765 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2766 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2767 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2768 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2769 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2770
2771 // Used for all non-volatile memory accesses. The use of
2772 // $mem->opcode() to discover whether this pattern uses sign-extended
2773 // offsets is something of a kludge.
2774 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2775 Register reg, int opcode,
2776 Register base, int index, int scale, int disp,
2777 int size_in_memory)
2778 {
2779 Address addr = mem2address(opcode, base, index, scale, disp);
2780 if (addr.getMode() == Address::base_plus_offset) {
2781 /* Fix up any out-of-range offsets. */
2782 assert_different_registers(rscratch1, base);
2783 assert_different_registers(rscratch1, reg);
2784 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2785 }
2786 (masm->*insn)(reg, addr);
2787 }
2788
2789 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2790 FloatRegister reg, int opcode,
2791 Register base, int index, int size, int disp,
2792 int size_in_memory)
2793 {
2794 Address::extend scale;
2795
2796 switch (opcode) {
2797 case INDINDEXSCALEDI2L:
2798 case INDINDEXSCALEDI2LN:
2799 scale = Address::sxtw(size);
2800 break;
2801 default:
2802 scale = Address::lsl(size);
2803 }
2804
2805 if (index == -1) {
2806 // Fix up any out-of-range offsets.
2807 assert_different_registers(rscratch1, base);
2808 Address addr = Address(base, disp);
2809 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2810 (masm->*insn)(reg, addr);
2811 } else {
2812 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2813 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2814 }
2815 }
2816
2817 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2818 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2819 int opcode, Register base, int index, int size, int disp)
2820 {
2821 if (index == -1) {
2822 (masm->*insn)(reg, T, Address(base, disp));
2823 } else {
2824 assert(disp == 0, "unsupported address mode");
2825 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2826 }
2827 }
2828
2829 %}
2830
2831
2832
2833 //----------ENCODING BLOCK-----------------------------------------------------
2834 // This block specifies the encoding classes used by the compiler to
2835 // output byte streams. Encoding classes are parameterized macros
2836 // used by Machine Instruction Nodes in order to generate the bit
2837 // encoding of the instruction. Operands specify their base encoding
2838 // interface with the interface keyword. There are currently
2839 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2840 // COND_INTER. REG_INTER causes an operand to generate a function
2841 // which returns its register number when queried. CONST_INTER causes
2842 // an operand to generate a function which returns the value of the
2843 // constant when queried. MEMORY_INTER causes an operand to generate
2844 // four functions which return the Base Register, the Index Register,
2845 // the Scale Value, and the Offset Value of the operand when queried.
2846 // COND_INTER causes an operand to generate six functions which return
2847 // the encoding code (ie - encoding bits for the instruction)
2848 // associated with each basic boolean condition for a conditional
2849 // instruction.
2850 //
2851 // Instructions specify two basic values for encoding. Again, a
2852 // function is available to check if the constant displacement is an
2853 // oop. They use the ins_encode keyword to specify their encoding
2854 // classes (which must be a sequence of enc_class names, and their
2855 // parameters, specified in the encoding block), and they use the
2856 // opcode keyword to specify, in order, their primary, secondary, and
2857 // tertiary opcode. Only the opcode sections which a particular
2858 // instruction needs for encoding need to be specified.
2859 encode %{
2860 // Build emit functions for each basic byte or larger field in the
2861 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2862 // from C++ code in the enc_class source block. Emit functions will
2863 // live in the main source block for now. In future, we can
2864 // generalize this by adding a syntax that specifies the sizes of
2865 // fields in an order, so that the adlc can build the emit functions
2866 // automagically
2867
2868 // catch all for unimplemented encodings
2869 enc_class enc_unimplemented %{
2870 __ unimplemented("C2 catch all");
2871 %}
2872
2873 // BEGIN Non-volatile memory access
2874
2875 // This encoding class is generated automatically from ad_encode.m4.
2876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2877 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2878 Register dst_reg = as_Register($dst$$reg);
2879 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2880 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2881 %}
2882
2883 // This encoding class is generated automatically from ad_encode.m4.
2884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2885 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2886 Register dst_reg = as_Register($dst$$reg);
2887 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2888 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2889 %}
2890
2891 // This encoding class is generated automatically from ad_encode.m4.
2892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2893 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2894 Register dst_reg = as_Register($dst$$reg);
2895 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2896 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2897 %}
2898
2899 // This encoding class is generated automatically from ad_encode.m4.
2900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2901 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2902 Register dst_reg = as_Register($dst$$reg);
2903 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2904 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2905 %}
2906
2907 // This encoding class is generated automatically from ad_encode.m4.
2908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2909 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2910 Register dst_reg = as_Register($dst$$reg);
2911 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2912 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2913 %}
2914
2915 // This encoding class is generated automatically from ad_encode.m4.
2916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2917 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2918 Register dst_reg = as_Register($dst$$reg);
2919 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2921 %}
2922
2923 // This encoding class is generated automatically from ad_encode.m4.
2924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2925 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2926 Register dst_reg = as_Register($dst$$reg);
2927 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2928 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2929 %}
2930
2931 // This encoding class is generated automatically from ad_encode.m4.
2932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2933 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2934 Register dst_reg = as_Register($dst$$reg);
2935 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2937 %}
2938
2939 // This encoding class is generated automatically from ad_encode.m4.
2940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2941 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2942 Register dst_reg = as_Register($dst$$reg);
2943 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2950 Register dst_reg = as_Register($dst$$reg);
2951 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2952 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2953 %}
2954
2955 // This encoding class is generated automatically from ad_encode.m4.
2956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2957 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2958 Register dst_reg = as_Register($dst$$reg);
2959 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2960 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2961 %}
2962
2963 // This encoding class is generated automatically from ad_encode.m4.
2964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2965 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2966 Register dst_reg = as_Register($dst$$reg);
2967 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2968 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2969 %}
2970
2971 // This encoding class is generated automatically from ad_encode.m4.
2972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2973 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2974 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2975 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2976 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2977 %}
2978
2979 // This encoding class is generated automatically from ad_encode.m4.
2980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2981 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2982 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2983 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2984 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2985 %}
2986
2987 // This encoding class is generated automatically from ad_encode.m4.
2988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2989 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2990 Register src_reg = as_Register($src$$reg);
2991 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2992 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2993 %}
2994
2995 // This encoding class is generated automatically from ad_encode.m4.
2996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2997 enc_class aarch64_enc_strb0(memory1 mem) %{
2998 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2999 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3000 %}
3001
3002 // This encoding class is generated automatically from ad_encode.m4.
3003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3004 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3005 Register src_reg = as_Register($src$$reg);
3006 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3007 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3008 %}
3009
3010 // This encoding class is generated automatically from ad_encode.m4.
3011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3012 enc_class aarch64_enc_strh0(memory2 mem) %{
3013 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3014 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3015 %}
3016
3017 // This encoding class is generated automatically from ad_encode.m4.
3018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3019 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3020 Register src_reg = as_Register($src$$reg);
3021 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3022 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3023 %}
3024
3025 // This encoding class is generated automatically from ad_encode.m4.
3026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3027 enc_class aarch64_enc_strw0(memory4 mem) %{
3028 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3029 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3030 %}
3031
3032 // This encoding class is generated automatically from ad_encode.m4.
3033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3034 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3035 Register src_reg = as_Register($src$$reg);
3036 // we sometimes get asked to store the stack pointer into the
3037 // current thread -- we cannot do that directly on AArch64
3038 if (src_reg == r31_sp) {
3039 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3040 __ mov(rscratch2, sp);
3041 src_reg = rscratch2;
3042 }
3043 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3044 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3045 %}
3046
3047 // This encoding class is generated automatically from ad_encode.m4.
3048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3049 enc_class aarch64_enc_str0(memory8 mem) %{
3050 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3051 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3052 %}
3053
3054 // This encoding class is generated automatically from ad_encode.m4.
3055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3056 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3057 FloatRegister src_reg = as_FloatRegister($src$$reg);
3058 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3059 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3060 %}
3061
3062 // This encoding class is generated automatically from ad_encode.m4.
3063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3064 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3065 FloatRegister src_reg = as_FloatRegister($src$$reg);
3066 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3067 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3068 %}
3069
3070 // This encoding class is generated automatically from ad_encode.m4.
3071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3072 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3073 __ membar(Assembler::StoreStore);
3074 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3075 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3076 %}
3077
3078 // END Non-volatile memory access
3079
3080 // Vector loads and stores
3081 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3082 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3083 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3084 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3085 %}
3086
3087 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3088 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3089 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3090 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3091 %}
3092
3093 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3094 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3095 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3096 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3097 %}
3098
3099 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3100 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3101 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3102 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3103 %}
3104
3105 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3106 FloatRegister src_reg = as_FloatRegister($src$$reg);
3107 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3108 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3109 %}
3110
3111 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3112 FloatRegister src_reg = as_FloatRegister($src$$reg);
3113 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3114 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3115 %}
3116
3117 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3118 FloatRegister src_reg = as_FloatRegister($src$$reg);
3119 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3120 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3121 %}
3122
3123 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3124 FloatRegister src_reg = as_FloatRegister($src$$reg);
3125 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3126 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3127 %}
3128
3129 // volatile loads and stores
3130
3131 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3132 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3133 rscratch1, stlrb);
3134 %}
3135
3136 enc_class aarch64_enc_stlrb0(memory mem) %{
3137 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3138 rscratch1, stlrb);
3139 %}
3140
3141 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3142 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3143 rscratch1, stlrh);
3144 %}
3145
3146 enc_class aarch64_enc_stlrh0(memory mem) %{
3147 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3148 rscratch1, stlrh);
3149 %}
3150
3151 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3152 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3153 rscratch1, stlrw);
3154 %}
3155
3156 enc_class aarch64_enc_stlrw0(memory mem) %{
3157 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3158 rscratch1, stlrw);
3159 %}
3160
3161 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3162 Register dst_reg = as_Register($dst$$reg);
3163 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3164 rscratch1, ldarb);
3165 __ sxtbw(dst_reg, dst_reg);
3166 %}
3167
3168 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3169 Register dst_reg = as_Register($dst$$reg);
3170 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3171 rscratch1, ldarb);
3172 __ sxtb(dst_reg, dst_reg);
3173 %}
3174
3175 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3176 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3177 rscratch1, ldarb);
3178 %}
3179
3180 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3181 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3182 rscratch1, ldarb);
3183 %}
3184
3185 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3186 Register dst_reg = as_Register($dst$$reg);
3187 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3188 rscratch1, ldarh);
3189 __ sxthw(dst_reg, dst_reg);
3190 %}
3191
3192 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3193 Register dst_reg = as_Register($dst$$reg);
3194 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3195 rscratch1, ldarh);
3196 __ sxth(dst_reg, dst_reg);
3197 %}
3198
3199 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3200 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3201 rscratch1, ldarh);
3202 %}
3203
3204 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3205 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3206 rscratch1, ldarh);
3207 %}
3208
3209 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3210 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3211 rscratch1, ldarw);
3212 %}
3213
3214 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3215 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3216 rscratch1, ldarw);
3217 %}
3218
3219 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3220 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3221 rscratch1, ldar);
3222 %}
3223
3224 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3225 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3226 rscratch1, ldarw);
3227 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3228 %}
3229
3230 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3231 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3232 rscratch1, ldar);
3233 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3234 %}
3235
3236 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3237 Register src_reg = as_Register($src$$reg);
3238 // we sometimes get asked to store the stack pointer into the
3239 // current thread -- we cannot do that directly on AArch64
3240 if (src_reg == r31_sp) {
3241 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3242 __ mov(rscratch2, sp);
3243 src_reg = rscratch2;
3244 }
3245 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3246 rscratch1, stlr);
3247 %}
3248
3249 enc_class aarch64_enc_stlr0(memory mem) %{
3250 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3251 rscratch1, stlr);
3252 %}
3253
3254 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3255 {
3256 FloatRegister src_reg = as_FloatRegister($src$$reg);
3257 __ fmovs(rscratch2, src_reg);
3258 }
3259 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3260 rscratch1, stlrw);
3261 %}
3262
3263 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3264 {
3265 FloatRegister src_reg = as_FloatRegister($src$$reg);
3266 __ fmovd(rscratch2, src_reg);
3267 }
3268 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3269 rscratch1, stlr);
3270 %}
3271
3272 // synchronized read/update encodings
3273
3274 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3275 Register dst_reg = as_Register($dst$$reg);
3276 Register base = as_Register($mem$$base);
3277 int index = $mem$$index;
3278 int scale = $mem$$scale;
3279 int disp = $mem$$disp;
3280 if (index == -1) {
3281 if (disp != 0) {
3282 __ lea(rscratch1, Address(base, disp));
3283 __ ldaxr(dst_reg, rscratch1);
3284 } else {
3285 // TODO
3286 // should we ever get anything other than this case?
3287 __ ldaxr(dst_reg, base);
3288 }
3289 } else {
3290 Register index_reg = as_Register(index);
3291 if (disp == 0) {
3292 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3293 __ ldaxr(dst_reg, rscratch1);
3294 } else {
3295 __ lea(rscratch1, Address(base, disp));
3296 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3297 __ ldaxr(dst_reg, rscratch1);
3298 }
3299 }
3300 %}
3301
3302 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3303 Register src_reg = as_Register($src$$reg);
3304 Register base = as_Register($mem$$base);
3305 int index = $mem$$index;
3306 int scale = $mem$$scale;
3307 int disp = $mem$$disp;
3308 if (index == -1) {
3309 if (disp != 0) {
3310 __ lea(rscratch2, Address(base, disp));
3311 __ stlxr(rscratch1, src_reg, rscratch2);
3312 } else {
3313 // TODO
3314 // should we ever get anything other than this case?
3315 __ stlxr(rscratch1, src_reg, base);
3316 }
3317 } else {
3318 Register index_reg = as_Register(index);
3319 if (disp == 0) {
3320 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3321 __ stlxr(rscratch1, src_reg, rscratch2);
3322 } else {
3323 __ lea(rscratch2, Address(base, disp));
3324 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3325 __ stlxr(rscratch1, src_reg, rscratch2);
3326 }
3327 }
3328 __ cmpw(rscratch1, zr);
3329 %}
3330
3331 // prefetch encodings
3332
3333 enc_class aarch64_enc_prefetchw(memory mem) %{
3334 Register base = as_Register($mem$$base);
3335 int index = $mem$$index;
3336 int scale = $mem$$scale;
3337 int disp = $mem$$disp;
3338 if (index == -1) {
3339 // Fix up any out-of-range offsets.
3340 assert_different_registers(rscratch1, base);
3341 Address addr = Address(base, disp);
3342 addr = __ legitimize_address(addr, 8, rscratch1);
3343 __ prfm(addr, PSTL1KEEP);
3344 } else {
3345 Register index_reg = as_Register(index);
3346 if (disp == 0) {
3347 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3348 } else {
3349 __ lea(rscratch1, Address(base, disp));
3350 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3351 }
3352 }
3353 %}
3354
3355 // mov encodings
3356
3357 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3358 uint32_t con = (uint32_t)$src$$constant;
3359 Register dst_reg = as_Register($dst$$reg);
3360 if (con == 0) {
3361 __ movw(dst_reg, zr);
3362 } else {
3363 __ movw(dst_reg, con);
3364 }
3365 %}
3366
3367 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3368 Register dst_reg = as_Register($dst$$reg);
3369 uint64_t con = (uint64_t)$src$$constant;
3370 if (con == 0) {
3371 __ mov(dst_reg, zr);
3372 } else {
3373 __ mov(dst_reg, con);
3374 }
3375 %}
3376
3377 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3378 Register dst_reg = as_Register($dst$$reg);
3379 address con = (address)$src$$constant;
3380 if (con == nullptr || con == (address)1) {
3381 ShouldNotReachHere();
3382 } else {
3383 relocInfo::relocType rtype = $src->constant_reloc();
3384 if (rtype == relocInfo::oop_type) {
3385 __ movoop(dst_reg, (jobject)con);
3386 } else if (rtype == relocInfo::metadata_type) {
3387 __ mov_metadata(dst_reg, (Metadata*)con);
3388 } else {
3389 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3390 // load fake address constants using a normal move
3391 if (! __ is_valid_AArch64_address(con) ||
3392 con < (address)(uintptr_t)os::vm_page_size()) {
3393 __ mov(dst_reg, con);
3394 } else {
3395 // no reloc so just use adrp and add
3396 uint64_t offset;
3397 __ adrp(dst_reg, con, offset);
3398 __ add(dst_reg, dst_reg, offset);
3399 }
3400 }
3401 }
3402 %}
3403
3404 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3405 Register dst_reg = as_Register($dst$$reg);
3406 __ mov(dst_reg, zr);
3407 %}
3408
3409 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3410 Register dst_reg = as_Register($dst$$reg);
3411 __ mov(dst_reg, (uint64_t)1);
3412 %}
3413
3414 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3415 Register dst_reg = as_Register($dst$$reg);
3416 address con = (address)$src$$constant;
3417 if (con == nullptr) {
3418 ShouldNotReachHere();
3419 } else {
3420 relocInfo::relocType rtype = $src->constant_reloc();
3421 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3422 __ set_narrow_oop(dst_reg, (jobject)con);
3423 }
3424 %}
3425
3426 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3427 Register dst_reg = as_Register($dst$$reg);
3428 __ mov(dst_reg, zr);
3429 %}
3430
3431 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3432 Register dst_reg = as_Register($dst$$reg);
3433 address con = (address)$src$$constant;
3434 if (con == nullptr) {
3435 ShouldNotReachHere();
3436 } else {
3437 relocInfo::relocType rtype = $src->constant_reloc();
3438 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3439 __ set_narrow_klass(dst_reg, (Klass *)con);
3440 }
3441 %}
3442
3443 // arithmetic encodings
3444
3445 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3446 Register dst_reg = as_Register($dst$$reg);
3447 Register src_reg = as_Register($src1$$reg);
3448 int32_t con = (int32_t)$src2$$constant;
3449 // add has primary == 0, subtract has primary == 1
3450 if ($primary) { con = -con; }
3451 if (con < 0) {
3452 __ subw(dst_reg, src_reg, -con);
3453 } else {
3454 __ addw(dst_reg, src_reg, con);
3455 }
3456 %}
3457
3458 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3459 Register dst_reg = as_Register($dst$$reg);
3460 Register src_reg = as_Register($src1$$reg);
3461 int32_t con = (int32_t)$src2$$constant;
3462 // add has primary == 0, subtract has primary == 1
3463 if ($primary) { con = -con; }
3464 if (con < 0) {
3465 __ sub(dst_reg, src_reg, -con);
3466 } else {
3467 __ add(dst_reg, src_reg, con);
3468 }
3469 %}
3470
3471 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3472 Register dst_reg = as_Register($dst$$reg);
3473 Register src1_reg = as_Register($src1$$reg);
3474 Register src2_reg = as_Register($src2$$reg);
3475 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3476 %}
3477
3478 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3479 Register dst_reg = as_Register($dst$$reg);
3480 Register src1_reg = as_Register($src1$$reg);
3481 Register src2_reg = as_Register($src2$$reg);
3482 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3483 %}
3484
3485 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3486 Register dst_reg = as_Register($dst$$reg);
3487 Register src1_reg = as_Register($src1$$reg);
3488 Register src2_reg = as_Register($src2$$reg);
3489 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3490 %}
3491
3492 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3493 Register dst_reg = as_Register($dst$$reg);
3494 Register src1_reg = as_Register($src1$$reg);
3495 Register src2_reg = as_Register($src2$$reg);
3496 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3497 %}
3498
3499 // compare instruction encodings
3500
3501 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3502 Register reg1 = as_Register($src1$$reg);
3503 Register reg2 = as_Register($src2$$reg);
3504 __ cmpw(reg1, reg2);
3505 %}
3506
3507 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3508 Register reg = as_Register($src1$$reg);
3509 int32_t val = $src2$$constant;
3510 if (val >= 0) {
3511 __ subsw(zr, reg, val);
3512 } else {
3513 __ addsw(zr, reg, -val);
3514 }
3515 %}
3516
3517 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3518 Register reg1 = as_Register($src1$$reg);
3519 uint32_t val = (uint32_t)$src2$$constant;
3520 __ movw(rscratch1, val);
3521 __ cmpw(reg1, rscratch1);
3522 %}
3523
3524 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3525 Register reg1 = as_Register($src1$$reg);
3526 Register reg2 = as_Register($src2$$reg);
3527 __ cmp(reg1, reg2);
3528 %}
3529
3530 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3531 Register reg = as_Register($src1$$reg);
3532 int64_t val = $src2$$constant;
3533 if (val >= 0) {
3534 __ subs(zr, reg, val);
3535 } else if (val != -val) {
3536 __ adds(zr, reg, -val);
3537 } else {
3538 // aargh, Long.MIN_VALUE is a special case
3539 __ orr(rscratch1, zr, (uint64_t)val);
3540 __ subs(zr, reg, rscratch1);
3541 }
3542 %}
3543
3544 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3545 Register reg1 = as_Register($src1$$reg);
3546 uint64_t val = (uint64_t)$src2$$constant;
3547 __ mov(rscratch1, val);
3548 __ cmp(reg1, rscratch1);
3549 %}
3550
3551 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3552 Register reg1 = as_Register($src1$$reg);
3553 Register reg2 = as_Register($src2$$reg);
3554 __ cmp(reg1, reg2);
3555 %}
3556
3557 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3558 Register reg1 = as_Register($src1$$reg);
3559 Register reg2 = as_Register($src2$$reg);
3560 __ cmpw(reg1, reg2);
3561 %}
3562
3563 enc_class aarch64_enc_testp(iRegP src) %{
3564 Register reg = as_Register($src$$reg);
3565 __ cmp(reg, zr);
3566 %}
3567
3568 enc_class aarch64_enc_testn(iRegN src) %{
3569 Register reg = as_Register($src$$reg);
3570 __ cmpw(reg, zr);
3571 %}
3572
3573 enc_class aarch64_enc_b(label lbl) %{
3574 Label *L = $lbl$$label;
3575 __ b(*L);
3576 %}
3577
3578 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3579 Label *L = $lbl$$label;
3580 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3581 %}
3582
3583 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3584 Label *L = $lbl$$label;
3585 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3586 %}
3587
3588 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3589 %{
3590 Register sub_reg = as_Register($sub$$reg);
3591 Register super_reg = as_Register($super$$reg);
3592 Register temp_reg = as_Register($temp$$reg);
3593 Register result_reg = as_Register($result$$reg);
3594
3595 Label miss;
3596 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3597 nullptr, &miss,
3598 /*set_cond_codes:*/ true);
3599 if ($primary) {
3600 __ mov(result_reg, zr);
3601 }
3602 __ bind(miss);
3603 %}
3604
3605 enc_class aarch64_enc_java_static_call(method meth) %{
3606 address addr = (address)$meth$$method;
3607 address call;
3608 if (!_method) {
3609 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3610 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3611 if (call == nullptr) {
3612 ciEnv::current()->record_failure("CodeCache is full");
3613 return;
3614 }
3615 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3616 // The NOP here is purely to ensure that eliding a call to
3617 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3618 __ nop();
3619 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3620 } else {
3621 int method_index = resolved_method_index(masm);
3622 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3623 : static_call_Relocation::spec(method_index);
3624 call = __ trampoline_call(Address(addr, rspec));
3625 if (call == nullptr) {
3626 ciEnv::current()->record_failure("CodeCache is full");
3627 return;
3628 }
3629 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3630 // Calls of the same statically bound method can share
3631 // a stub to the interpreter.
3632 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3633 } else {
3634 // Emit stub for static call
3635 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3636 if (stub == nullptr) {
3637 ciEnv::current()->record_failure("CodeCache is full");
3638 return;
3639 }
3640 }
3641 }
3642
3643 __ post_call_nop();
3644
3645 // Only non uncommon_trap calls need to reinitialize ptrue.
3646 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3647 __ reinitialize_ptrue();
3648 }
3649 %}
3650
3651 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3652 int method_index = resolved_method_index(masm);
3653 address call = __ ic_call((address)$meth$$method, method_index);
3654 if (call == nullptr) {
3655 ciEnv::current()->record_failure("CodeCache is full");
3656 return;
3657 }
3658 __ post_call_nop();
3659 if (Compile::current()->max_vector_size() > 0) {
3660 __ reinitialize_ptrue();
3661 }
3662 %}
3663
3664 enc_class aarch64_enc_call_epilog() %{
3665 if (VerifyStackAtCalls) {
3666 // Check that stack depth is unchanged: find majik cookie on stack
3667 __ call_Unimplemented();
3668 }
3669 %}
3670
3671 enc_class aarch64_enc_java_to_runtime(method meth) %{
3672 // some calls to generated routines (arraycopy code) are scheduled
3673 // by C2 as runtime calls. if so we can call them using a br (they
3674 // will be in a reachable segment) otherwise we have to use a blr
3675 // which loads the absolute address into a register.
3676 address entry = (address)$meth$$method;
3677 CodeBlob *cb = CodeCache::find_blob(entry);
3678 if (cb) {
3679 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3680 if (call == nullptr) {
3681 ciEnv::current()->record_failure("CodeCache is full");
3682 return;
3683 }
3684 __ post_call_nop();
3685 } else {
3686 Label retaddr;
3687 // Make the anchor frame walkable
3688 __ adr(rscratch2, retaddr);
3689 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3690 __ lea(rscratch1, RuntimeAddress(entry));
3691 __ blr(rscratch1);
3692 __ bind(retaddr);
3693 __ post_call_nop();
3694 }
3695 if (Compile::current()->max_vector_size() > 0) {
3696 __ reinitialize_ptrue();
3697 }
3698 %}
3699
3700 enc_class aarch64_enc_rethrow() %{
3701 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3702 %}
3703
3704 enc_class aarch64_enc_ret() %{
3705 #ifdef ASSERT
3706 if (Compile::current()->max_vector_size() > 0) {
3707 __ verify_ptrue();
3708 }
3709 #endif
3710 __ ret(lr);
3711 %}
3712
3713 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3714 Register target_reg = as_Register($jump_target$$reg);
3715 __ br(target_reg);
3716 %}
3717
3718 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3719 Register target_reg = as_Register($jump_target$$reg);
3720 // exception oop should be in r0
3721 // ret addr has been popped into lr
3722 // callee expects it in r3
3723 __ mov(r3, lr);
3724 __ br(target_reg);
3725 %}
3726
3727 %}
3728
3729 //----------FRAME--------------------------------------------------------------
3730 // Definition of frame structure and management information.
3731 //
3732 // S T A C K L A Y O U T Allocators stack-slot number
3733 // | (to get allocators register number
3734 // G Owned by | | v add OptoReg::stack0())
3735 // r CALLER | |
3736 // o | +--------+ pad to even-align allocators stack-slot
3737 // w V | pad0 | numbers; owned by CALLER
3738 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3739 // h ^ | in | 5
3740 // | | args | 4 Holes in incoming args owned by SELF
3741 // | | | | 3
3742 // | | +--------+
3743 // V | | old out| Empty on Intel, window on Sparc
3744 // | old |preserve| Must be even aligned.
3745 // | SP-+--------+----> Matcher::_old_SP, even aligned
3746 // | | in | 3 area for Intel ret address
3747 // Owned by |preserve| Empty on Sparc.
3748 // SELF +--------+
3749 // | | pad2 | 2 pad to align old SP
3750 // | +--------+ 1
3751 // | | locks | 0
3752 // | +--------+----> OptoReg::stack0(), even aligned
3753 // | | pad1 | 11 pad to align new SP
3754 // | +--------+
3755 // | | | 10
3756 // | | spills | 9 spills
3757 // V | | 8 (pad0 slot for callee)
3758 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3759 // ^ | out | 7
3760 // | | args | 6 Holes in outgoing args owned by CALLEE
3761 // Owned by +--------+
3762 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3763 // | new |preserve| Must be even-aligned.
3764 // | SP-+--------+----> Matcher::_new_SP, even aligned
3765 // | | |
3766 //
3767 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3768 // known from SELF's arguments and the Java calling convention.
3769 // Region 6-7 is determined per call site.
3770 // Note 2: If the calling convention leaves holes in the incoming argument
3771 // area, those holes are owned by SELF. Holes in the outgoing area
3772 // are owned by the CALLEE. Holes should not be necessary in the
3773 // incoming area, as the Java calling convention is completely under
3774 // the control of the AD file. Doubles can be sorted and packed to
3775 // avoid holes. Holes in the outgoing arguments may be necessary for
3776 // varargs C calling conventions.
3777 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3778 // even aligned with pad0 as needed.
3779 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3780 // (the latter is true on Intel but is it false on AArch64?)
3781 // region 6-11 is even aligned; it may be padded out more so that
3782 // the region from SP to FP meets the minimum stack alignment.
3783 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3784 // alignment. Region 11, pad1, may be dynamically extended so that
3785 // SP meets the minimum alignment.
3786
3787 frame %{
3788 // These three registers define part of the calling convention
3789 // between compiled code and the interpreter.
3790
3791 // Inline Cache Register or Method for I2C.
3792 inline_cache_reg(R12);
3793
3794 // Number of stack slots consumed by locking an object
3795 sync_stack_slots(2);
3796
3797 // Compiled code's Frame Pointer
3798 frame_pointer(R31);
3799
3800 // Stack alignment requirement
3801 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3802
3803 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3804 // for calls to C. Supports the var-args backing area for register parms.
3805 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3806
3807 // The after-PROLOG location of the return address. Location of
3808 // return address specifies a type (REG or STACK) and a number
3809 // representing the register number (i.e. - use a register name) or
3810 // stack slot.
3811 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3812 // Otherwise, it is above the locks and verification slot and alignment word
3813 // TODO this may well be correct but need to check why that - 2 is there
3814 // ppc port uses 0 but we definitely need to allow for fixed_slots
3815 // which folds in the space used for monitors
3816 return_addr(STACK - 2 +
3817 align_up((Compile::current()->in_preserve_stack_slots() +
3818 Compile::current()->fixed_slots()),
3819 stack_alignment_in_slots()));
3820
3821 // Location of compiled Java return values. Same as C for now.
3822 return_value
3823 %{
3824 // TODO do we allow ideal_reg == Op_RegN???
3825 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3826 "only return normal values");
3827
3828 static const int lo[Op_RegL + 1] = { // enum name
3829 0, // Op_Node
3830 0, // Op_Set
3831 R0_num, // Op_RegN
3832 R0_num, // Op_RegI
3833 R0_num, // Op_RegP
3834 V0_num, // Op_RegF
3835 V0_num, // Op_RegD
3836 R0_num // Op_RegL
3837 };
3838
3839 static const int hi[Op_RegL + 1] = { // enum name
3840 0, // Op_Node
3841 0, // Op_Set
3842 OptoReg::Bad, // Op_RegN
3843 OptoReg::Bad, // Op_RegI
3844 R0_H_num, // Op_RegP
3845 OptoReg::Bad, // Op_RegF
3846 V0_H_num, // Op_RegD
3847 R0_H_num // Op_RegL
3848 };
3849
3850 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3851 %}
3852 %}
3853
3854 //----------ATTRIBUTES---------------------------------------------------------
3855 //----------Operand Attributes-------------------------------------------------
3856 op_attrib op_cost(1); // Required cost attribute
3857
3858 //----------Instruction Attributes---------------------------------------------
3859 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3860 ins_attrib ins_size(32); // Required size attribute (in bits)
3861 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3862 // a non-matching short branch variant
3863 // of some long branch?
3864 ins_attrib ins_alignment(4); // Required alignment attribute (must
3865 // be a power of 2) specifies the
3866 // alignment that some part of the
3867 // instruction (not necessarily the
3868 // start) requires. If > 1, a
3869 // compute_padding() function must be
3870 // provided for the instruction
3871
3872 // Whether this node is expanded during code emission into a sequence of
3873 // instructions and the first instruction can perform an implicit null check.
3874 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3875
3876 //----------OPERANDS-----------------------------------------------------------
3877 // Operand definitions must precede instruction definitions for correct parsing
3878 // in the ADLC because operands constitute user defined types which are used in
3879 // instruction definitions.
3880
3881 //----------Simple Operands----------------------------------------------------
3882
3883 // Integer operands 32 bit
3884 // 32 bit immediate
3885 operand immI()
3886 %{
3887 match(ConI);
3888
3889 op_cost(0);
3890 format %{ %}
3891 interface(CONST_INTER);
3892 %}
3893
3894 // 32 bit zero
3895 operand immI0()
3896 %{
3897 predicate(n->get_int() == 0);
3898 match(ConI);
3899
3900 op_cost(0);
3901 format %{ %}
3902 interface(CONST_INTER);
3903 %}
3904
3905 // 32 bit unit increment
3906 operand immI_1()
3907 %{
3908 predicate(n->get_int() == 1);
3909 match(ConI);
3910
3911 op_cost(0);
3912 format %{ %}
3913 interface(CONST_INTER);
3914 %}
3915
3916 // 32 bit unit decrement
3917 operand immI_M1()
3918 %{
3919 predicate(n->get_int() == -1);
3920 match(ConI);
3921
3922 op_cost(0);
3923 format %{ %}
3924 interface(CONST_INTER);
3925 %}
3926
3927 // Shift values for add/sub extension shift
3928 operand immIExt()
3929 %{
3930 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3931 match(ConI);
3932
3933 op_cost(0);
3934 format %{ %}
3935 interface(CONST_INTER);
3936 %}
3937
3938 operand immI_gt_1()
3939 %{
3940 predicate(n->get_int() > 1);
3941 match(ConI);
3942
3943 op_cost(0);
3944 format %{ %}
3945 interface(CONST_INTER);
3946 %}
3947
3948 operand immI_le_4()
3949 %{
3950 predicate(n->get_int() <= 4);
3951 match(ConI);
3952
3953 op_cost(0);
3954 format %{ %}
3955 interface(CONST_INTER);
3956 %}
3957
3958 operand immI_16()
3959 %{
3960 predicate(n->get_int() == 16);
3961 match(ConI);
3962
3963 op_cost(0);
3964 format %{ %}
3965 interface(CONST_INTER);
3966 %}
3967
3968 operand immI_24()
3969 %{
3970 predicate(n->get_int() == 24);
3971 match(ConI);
3972
3973 op_cost(0);
3974 format %{ %}
3975 interface(CONST_INTER);
3976 %}
3977
3978 operand immI_32()
3979 %{
3980 predicate(n->get_int() == 32);
3981 match(ConI);
3982
3983 op_cost(0);
3984 format %{ %}
3985 interface(CONST_INTER);
3986 %}
3987
3988 operand immI_48()
3989 %{
3990 predicate(n->get_int() == 48);
3991 match(ConI);
3992
3993 op_cost(0);
3994 format %{ %}
3995 interface(CONST_INTER);
3996 %}
3997
3998 operand immI_56()
3999 %{
4000 predicate(n->get_int() == 56);
4001 match(ConI);
4002
4003 op_cost(0);
4004 format %{ %}
4005 interface(CONST_INTER);
4006 %}
4007
4008 operand immI_255()
4009 %{
4010 predicate(n->get_int() == 255);
4011 match(ConI);
4012
4013 op_cost(0);
4014 format %{ %}
4015 interface(CONST_INTER);
4016 %}
4017
4018 operand immI_65535()
4019 %{
4020 predicate(n->get_int() == 65535);
4021 match(ConI);
4022
4023 op_cost(0);
4024 format %{ %}
4025 interface(CONST_INTER);
4026 %}
4027
4028 operand immI_positive()
4029 %{
4030 predicate(n->get_int() > 0);
4031 match(ConI);
4032
4033 op_cost(0);
4034 format %{ %}
4035 interface(CONST_INTER);
4036 %}
4037
4038 // BoolTest condition for signed compare
4039 operand immI_cmp_cond()
4040 %{
4041 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4042 match(ConI);
4043
4044 op_cost(0);
4045 format %{ %}
4046 interface(CONST_INTER);
4047 %}
4048
4049 // BoolTest condition for unsigned compare
4050 operand immI_cmpU_cond()
4051 %{
4052 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4053 match(ConI);
4054
4055 op_cost(0);
4056 format %{ %}
4057 interface(CONST_INTER);
4058 %}
4059
4060 operand immL_255()
4061 %{
4062 predicate(n->get_long() == 255L);
4063 match(ConL);
4064
4065 op_cost(0);
4066 format %{ %}
4067 interface(CONST_INTER);
4068 %}
4069
4070 operand immL_65535()
4071 %{
4072 predicate(n->get_long() == 65535L);
4073 match(ConL);
4074
4075 op_cost(0);
4076 format %{ %}
4077 interface(CONST_INTER);
4078 %}
4079
4080 operand immL_4294967295()
4081 %{
4082 predicate(n->get_long() == 4294967295L);
4083 match(ConL);
4084
4085 op_cost(0);
4086 format %{ %}
4087 interface(CONST_INTER);
4088 %}
4089
4090 operand immL_bitmask()
4091 %{
4092 predicate((n->get_long() != 0)
4093 && ((n->get_long() & 0xc000000000000000l) == 0)
4094 && is_power_of_2(n->get_long() + 1));
4095 match(ConL);
4096
4097 op_cost(0);
4098 format %{ %}
4099 interface(CONST_INTER);
4100 %}
4101
4102 operand immI_bitmask()
4103 %{
4104 predicate((n->get_int() != 0)
4105 && ((n->get_int() & 0xc0000000) == 0)
4106 && is_power_of_2(n->get_int() + 1));
4107 match(ConI);
4108
4109 op_cost(0);
4110 format %{ %}
4111 interface(CONST_INTER);
4112 %}
4113
4114 operand immL_positive_bitmaskI()
4115 %{
4116 predicate((n->get_long() != 0)
4117 && ((julong)n->get_long() < 0x80000000ULL)
4118 && is_power_of_2(n->get_long() + 1));
4119 match(ConL);
4120
4121 op_cost(0);
4122 format %{ %}
4123 interface(CONST_INTER);
4124 %}
4125
4126 // Scale values for scaled offset addressing modes (up to long but not quad)
4127 operand immIScale()
4128 %{
4129 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4130 match(ConI);
4131
4132 op_cost(0);
4133 format %{ %}
4134 interface(CONST_INTER);
4135 %}
4136
4137 // 5 bit signed integer
4138 operand immI5()
4139 %{
4140 predicate(Assembler::is_simm(n->get_int(), 5));
4141 match(ConI);
4142
4143 op_cost(0);
4144 format %{ %}
4145 interface(CONST_INTER);
4146 %}
4147
4148 // 7 bit unsigned integer
4149 operand immIU7()
4150 %{
4151 predicate(Assembler::is_uimm(n->get_int(), 7));
4152 match(ConI);
4153
4154 op_cost(0);
4155 format %{ %}
4156 interface(CONST_INTER);
4157 %}
4158
4159 // Offset for scaled or unscaled immediate loads and stores
4160 operand immIOffset()
4161 %{
4162 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4163 match(ConI);
4164
4165 op_cost(0);
4166 format %{ %}
4167 interface(CONST_INTER);
4168 %}
4169
4170 operand immIOffset1()
4171 %{
4172 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4173 match(ConI);
4174
4175 op_cost(0);
4176 format %{ %}
4177 interface(CONST_INTER);
4178 %}
4179
4180 operand immIOffset2()
4181 %{
4182 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4183 match(ConI);
4184
4185 op_cost(0);
4186 format %{ %}
4187 interface(CONST_INTER);
4188 %}
4189
4190 operand immIOffset4()
4191 %{
4192 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4193 match(ConI);
4194
4195 op_cost(0);
4196 format %{ %}
4197 interface(CONST_INTER);
4198 %}
4199
4200 operand immIOffset8()
4201 %{
4202 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4203 match(ConI);
4204
4205 op_cost(0);
4206 format %{ %}
4207 interface(CONST_INTER);
4208 %}
4209
4210 operand immIOffset16()
4211 %{
4212 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4213 match(ConI);
4214
4215 op_cost(0);
4216 format %{ %}
4217 interface(CONST_INTER);
4218 %}
4219
4220 operand immLOffset()
4221 %{
4222 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4223 match(ConL);
4224
4225 op_cost(0);
4226 format %{ %}
4227 interface(CONST_INTER);
4228 %}
4229
4230 operand immLoffset1()
4231 %{
4232 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4233 match(ConL);
4234
4235 op_cost(0);
4236 format %{ %}
4237 interface(CONST_INTER);
4238 %}
4239
4240 operand immLoffset2()
4241 %{
4242 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4243 match(ConL);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 operand immLoffset4()
4251 %{
4252 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4253 match(ConL);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 operand immLoffset8()
4261 %{
4262 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4263 match(ConL);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immLoffset16()
4271 %{
4272 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4273 match(ConL);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 // 5 bit signed long integer
4281 operand immL5()
4282 %{
4283 predicate(Assembler::is_simm(n->get_long(), 5));
4284 match(ConL);
4285
4286 op_cost(0);
4287 format %{ %}
4288 interface(CONST_INTER);
4289 %}
4290
4291 // 7 bit unsigned long integer
4292 operand immLU7()
4293 %{
4294 predicate(Assembler::is_uimm(n->get_long(), 7));
4295 match(ConL);
4296
4297 op_cost(0);
4298 format %{ %}
4299 interface(CONST_INTER);
4300 %}
4301
4302 // 8 bit signed value.
4303 operand immI8()
4304 %{
4305 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4306 match(ConI);
4307
4308 op_cost(0);
4309 format %{ %}
4310 interface(CONST_INTER);
4311 %}
4312
4313 // 8 bit signed value (simm8), or #simm8 LSL 8.
4314 operand immIDupV()
4315 %{
4316 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4317 match(ConI);
4318
4319 op_cost(0);
4320 format %{ %}
4321 interface(CONST_INTER);
4322 %}
4323
4324 // 8 bit signed value (simm8), or #simm8 LSL 8.
4325 operand immLDupV()
4326 %{
4327 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4328 match(ConL);
4329
4330 op_cost(0);
4331 format %{ %}
4332 interface(CONST_INTER);
4333 %}
4334
4335 // 8 bit signed value (simm8), or #simm8 LSL 8.
4336 operand immHDupV()
4337 %{
4338 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4339 match(ConH);
4340
4341 op_cost(0);
4342 format %{ %}
4343 interface(CONST_INTER);
4344 %}
4345
4346 // 8 bit integer valid for vector add sub immediate
4347 operand immBAddSubV()
4348 %{
4349 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4350 match(ConI);
4351
4352 op_cost(0);
4353 format %{ %}
4354 interface(CONST_INTER);
4355 %}
4356
4357 // 32 bit integer valid for add sub immediate
4358 operand immIAddSub()
4359 %{
4360 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4361 match(ConI);
4362 op_cost(0);
4363 format %{ %}
4364 interface(CONST_INTER);
4365 %}
4366
4367 // 32 bit integer valid for vector add sub immediate
4368 operand immIAddSubV()
4369 %{
4370 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4371 match(ConI);
4372
4373 op_cost(0);
4374 format %{ %}
4375 interface(CONST_INTER);
4376 %}
4377
4378 // 32 bit unsigned integer valid for logical immediate
4379
4380 operand immBLog()
4381 %{
4382 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4383 match(ConI);
4384
4385 op_cost(0);
4386 format %{ %}
4387 interface(CONST_INTER);
4388 %}
4389
4390 operand immSLog()
4391 %{
4392 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4393 match(ConI);
4394
4395 op_cost(0);
4396 format %{ %}
4397 interface(CONST_INTER);
4398 %}
4399
4400 operand immILog()
4401 %{
4402 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4403 match(ConI);
4404
4405 op_cost(0);
4406 format %{ %}
4407 interface(CONST_INTER);
4408 %}
4409
4410 // Integer operands 64 bit
4411 // 64 bit immediate
4412 operand immL()
4413 %{
4414 match(ConL);
4415
4416 op_cost(0);
4417 format %{ %}
4418 interface(CONST_INTER);
4419 %}
4420
4421 // 64 bit zero
4422 operand immL0()
4423 %{
4424 predicate(n->get_long() == 0);
4425 match(ConL);
4426
4427 op_cost(0);
4428 format %{ %}
4429 interface(CONST_INTER);
4430 %}
4431
4432 // 64 bit unit decrement
4433 operand immL_M1()
4434 %{
4435 predicate(n->get_long() == -1);
4436 match(ConL);
4437
4438 op_cost(0);
4439 format %{ %}
4440 interface(CONST_INTER);
4441 %}
4442
4443 // 64 bit integer valid for add sub immediate
4444 operand immLAddSub()
4445 %{
4446 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4447 match(ConL);
4448 op_cost(0);
4449 format %{ %}
4450 interface(CONST_INTER);
4451 %}
4452
4453 // 64 bit integer valid for addv subv immediate
4454 operand immLAddSubV()
4455 %{
4456 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4457 match(ConL);
4458
4459 op_cost(0);
4460 format %{ %}
4461 interface(CONST_INTER);
4462 %}
4463
4464 // 64 bit integer valid for logical immediate
4465 operand immLLog()
4466 %{
4467 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4468 match(ConL);
4469 op_cost(0);
4470 format %{ %}
4471 interface(CONST_INTER);
4472 %}
4473
4474 // Long Immediate: low 32-bit mask
4475 operand immL_32bits()
4476 %{
4477 predicate(n->get_long() == 0xFFFFFFFFL);
4478 match(ConL);
4479 op_cost(0);
4480 format %{ %}
4481 interface(CONST_INTER);
4482 %}
4483
4484 // Pointer operands
4485 // Pointer Immediate
4486 operand immP()
4487 %{
4488 match(ConP);
4489
4490 op_cost(0);
4491 format %{ %}
4492 interface(CONST_INTER);
4493 %}
4494
4495 // nullptr Pointer Immediate
4496 operand immP0()
4497 %{
4498 predicate(n->get_ptr() == 0);
4499 match(ConP);
4500
4501 op_cost(0);
4502 format %{ %}
4503 interface(CONST_INTER);
4504 %}
4505
4506 // Pointer Immediate One
4507 // this is used in object initialization (initial object header)
4508 operand immP_1()
4509 %{
4510 predicate(n->get_ptr() == 1);
4511 match(ConP);
4512
4513 op_cost(0);
4514 format %{ %}
4515 interface(CONST_INTER);
4516 %}
4517
4518 // AOT Runtime Constants Address
4519 operand immAOTRuntimeConstantsAddress()
4520 %{
4521 // Check if the address is in the range of AOT Runtime Constants
4522 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4523 match(ConP);
4524
4525 op_cost(0);
4526 format %{ %}
4527 interface(CONST_INTER);
4528 %}
4529
4530 // Float and Double operands
4531 // Double Immediate
4532 operand immD()
4533 %{
4534 match(ConD);
4535 op_cost(0);
4536 format %{ %}
4537 interface(CONST_INTER);
4538 %}
4539
4540 // Double Immediate: +0.0d
4541 operand immD0()
4542 %{
4543 predicate(jlong_cast(n->getd()) == 0);
4544 match(ConD);
4545
4546 op_cost(0);
4547 format %{ %}
4548 interface(CONST_INTER);
4549 %}
4550
4551 // constant 'double +0.0'.
4552 operand immDPacked()
4553 %{
4554 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4555 match(ConD);
4556 op_cost(0);
4557 format %{ %}
4558 interface(CONST_INTER);
4559 %}
4560
4561 // Float Immediate
4562 operand immF()
4563 %{
4564 match(ConF);
4565 op_cost(0);
4566 format %{ %}
4567 interface(CONST_INTER);
4568 %}
4569
4570 // Float Immediate: +0.0f.
4571 operand immF0()
4572 %{
4573 predicate(jint_cast(n->getf()) == 0);
4574 match(ConF);
4575
4576 op_cost(0);
4577 format %{ %}
4578 interface(CONST_INTER);
4579 %}
4580
4581 // Half Float (FP16) Immediate
4582 operand immH()
4583 %{
4584 match(ConH);
4585 op_cost(0);
4586 format %{ %}
4587 interface(CONST_INTER);
4588 %}
4589
4590 //
4591 operand immFPacked()
4592 %{
4593 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4594 match(ConF);
4595 op_cost(0);
4596 format %{ %}
4597 interface(CONST_INTER);
4598 %}
4599
4600 // Narrow pointer operands
4601 // Narrow Pointer Immediate
4602 operand immN()
4603 %{
4604 match(ConN);
4605
4606 op_cost(0);
4607 format %{ %}
4608 interface(CONST_INTER);
4609 %}
4610
4611 // Narrow nullptr Pointer Immediate
4612 operand immN0()
4613 %{
4614 predicate(n->get_narrowcon() == 0);
4615 match(ConN);
4616
4617 op_cost(0);
4618 format %{ %}
4619 interface(CONST_INTER);
4620 %}
4621
4622 operand immNKlass()
4623 %{
4624 match(ConNKlass);
4625
4626 op_cost(0);
4627 format %{ %}
4628 interface(CONST_INTER);
4629 %}
4630
4631 // Integer 32 bit Register Operands
4632 // Integer 32 bitRegister (excludes SP)
4633 operand iRegI()
4634 %{
4635 constraint(ALLOC_IN_RC(any_reg32));
4636 match(RegI);
4637 match(iRegINoSp);
4638 op_cost(0);
4639 format %{ %}
4640 interface(REG_INTER);
4641 %}
4642
4643 // Integer 32 bit Register not Special
4644 operand iRegINoSp()
4645 %{
4646 constraint(ALLOC_IN_RC(no_special_reg32));
4647 match(RegI);
4648 op_cost(0);
4649 format %{ %}
4650 interface(REG_INTER);
4651 %}
4652
4653 // Integer 64 bit Register Operands
4654 // Integer 64 bit Register (includes SP)
4655 operand iRegL()
4656 %{
4657 constraint(ALLOC_IN_RC(any_reg));
4658 match(RegL);
4659 match(iRegLNoSp);
4660 op_cost(0);
4661 format %{ %}
4662 interface(REG_INTER);
4663 %}
4664
4665 // Integer 64 bit Register not Special
4666 operand iRegLNoSp()
4667 %{
4668 constraint(ALLOC_IN_RC(no_special_reg));
4669 match(RegL);
4670 match(iRegL_R0);
4671 format %{ %}
4672 interface(REG_INTER);
4673 %}
4674
4675 // Pointer Register Operands
4676 // Pointer Register
4677 operand iRegP()
4678 %{
4679 constraint(ALLOC_IN_RC(ptr_reg));
4680 match(RegP);
4681 match(iRegPNoSp);
4682 match(iRegP_R0);
4683 //match(iRegP_R2);
4684 //match(iRegP_R4);
4685 match(iRegP_R5);
4686 match(thread_RegP);
4687 op_cost(0);
4688 format %{ %}
4689 interface(REG_INTER);
4690 %}
4691
4692 // Pointer 64 bit Register not Special
4693 operand iRegPNoSp()
4694 %{
4695 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4696 match(RegP);
4697 // match(iRegP);
4698 // match(iRegP_R0);
4699 // match(iRegP_R2);
4700 // match(iRegP_R4);
4701 // match(iRegP_R5);
4702 // match(thread_RegP);
4703 op_cost(0);
4704 format %{ %}
4705 interface(REG_INTER);
4706 %}
4707
4708 // This operand is not allowed to use rfp even if
4709 // rfp is not used to hold the frame pointer.
4710 operand iRegPNoSpNoRfp()
4711 %{
4712 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4713 match(RegP);
4714 match(iRegPNoSp);
4715 op_cost(0);
4716 format %{ %}
4717 interface(REG_INTER);
4718 %}
4719
4720 // Pointer 64 bit Register R0 only
4721 operand iRegP_R0()
4722 %{
4723 constraint(ALLOC_IN_RC(r0_reg));
4724 match(RegP);
4725 // match(iRegP);
4726 match(iRegPNoSp);
4727 op_cost(0);
4728 format %{ %}
4729 interface(REG_INTER);
4730 %}
4731
4732 // Pointer 64 bit Register R1 only
4733 operand iRegP_R1()
4734 %{
4735 constraint(ALLOC_IN_RC(r1_reg));
4736 match(RegP);
4737 // match(iRegP);
4738 match(iRegPNoSp);
4739 op_cost(0);
4740 format %{ %}
4741 interface(REG_INTER);
4742 %}
4743
4744 // Pointer 64 bit Register R2 only
4745 operand iRegP_R2()
4746 %{
4747 constraint(ALLOC_IN_RC(r2_reg));
4748 match(RegP);
4749 // match(iRegP);
4750 match(iRegPNoSp);
4751 op_cost(0);
4752 format %{ %}
4753 interface(REG_INTER);
4754 %}
4755
4756 // Pointer 64 bit Register R3 only
4757 operand iRegP_R3()
4758 %{
4759 constraint(ALLOC_IN_RC(r3_reg));
4760 match(RegP);
4761 // match(iRegP);
4762 match(iRegPNoSp);
4763 op_cost(0);
4764 format %{ %}
4765 interface(REG_INTER);
4766 %}
4767
4768 // Pointer 64 bit Register R4 only
4769 operand iRegP_R4()
4770 %{
4771 constraint(ALLOC_IN_RC(r4_reg));
4772 match(RegP);
4773 // match(iRegP);
4774 match(iRegPNoSp);
4775 op_cost(0);
4776 format %{ %}
4777 interface(REG_INTER);
4778 %}
4779
4780 // Pointer 64 bit Register R5 only
4781 operand iRegP_R5()
4782 %{
4783 constraint(ALLOC_IN_RC(r5_reg));
4784 match(RegP);
4785 // match(iRegP);
4786 match(iRegPNoSp);
4787 op_cost(0);
4788 format %{ %}
4789 interface(REG_INTER);
4790 %}
4791
4792 // Pointer 64 bit Register R10 only
4793 operand iRegP_R10()
4794 %{
4795 constraint(ALLOC_IN_RC(r10_reg));
4796 match(RegP);
4797 // match(iRegP);
4798 match(iRegPNoSp);
4799 op_cost(0);
4800 format %{ %}
4801 interface(REG_INTER);
4802 %}
4803
4804 // Long 64 bit Register R0 only
4805 operand iRegL_R0()
4806 %{
4807 constraint(ALLOC_IN_RC(r0_reg));
4808 match(RegL);
4809 match(iRegLNoSp);
4810 op_cost(0);
4811 format %{ %}
4812 interface(REG_INTER);
4813 %}
4814
4815 // Long 64 bit Register R11 only
4816 operand iRegL_R11()
4817 %{
4818 constraint(ALLOC_IN_RC(r11_reg));
4819 match(RegL);
4820 match(iRegLNoSp);
4821 op_cost(0);
4822 format %{ %}
4823 interface(REG_INTER);
4824 %}
4825
4826 // Register R0 only
4827 operand iRegI_R0()
4828 %{
4829 constraint(ALLOC_IN_RC(int_r0_reg));
4830 match(RegI);
4831 match(iRegINoSp);
4832 op_cost(0);
4833 format %{ %}
4834 interface(REG_INTER);
4835 %}
4836
4837 // Register R2 only
4838 operand iRegI_R2()
4839 %{
4840 constraint(ALLOC_IN_RC(int_r2_reg));
4841 match(RegI);
4842 match(iRegINoSp);
4843 op_cost(0);
4844 format %{ %}
4845 interface(REG_INTER);
4846 %}
4847
4848 // Register R3 only
4849 operand iRegI_R3()
4850 %{
4851 constraint(ALLOC_IN_RC(int_r3_reg));
4852 match(RegI);
4853 match(iRegINoSp);
4854 op_cost(0);
4855 format %{ %}
4856 interface(REG_INTER);
4857 %}
4858
4859
4860 // Register R4 only
4861 operand iRegI_R4()
4862 %{
4863 constraint(ALLOC_IN_RC(int_r4_reg));
4864 match(RegI);
4865 match(iRegINoSp);
4866 op_cost(0);
4867 format %{ %}
4868 interface(REG_INTER);
4869 %}
4870
4871
4872 // Pointer Register Operands
4873 // Narrow Pointer Register
4874 operand iRegN()
4875 %{
4876 constraint(ALLOC_IN_RC(any_reg32));
4877 match(RegN);
4878 match(iRegNNoSp);
4879 op_cost(0);
4880 format %{ %}
4881 interface(REG_INTER);
4882 %}
4883
4884 // Integer 64 bit Register not Special
4885 operand iRegNNoSp()
4886 %{
4887 constraint(ALLOC_IN_RC(no_special_reg32));
4888 match(RegN);
4889 op_cost(0);
4890 format %{ %}
4891 interface(REG_INTER);
4892 %}
4893
4894 // Float Register
4895 // Float register operands
4896 operand vRegF()
4897 %{
4898 constraint(ALLOC_IN_RC(float_reg));
4899 match(RegF);
4900
4901 op_cost(0);
4902 format %{ %}
4903 interface(REG_INTER);
4904 %}
4905
4906 // Double Register
4907 // Double register operands
4908 operand vRegD()
4909 %{
4910 constraint(ALLOC_IN_RC(double_reg));
4911 match(RegD);
4912
4913 op_cost(0);
4914 format %{ %}
4915 interface(REG_INTER);
4916 %}
4917
4918 // Generic vector class. This will be used for
4919 // all vector operands, including NEON and SVE.
4920 operand vReg()
4921 %{
4922 constraint(ALLOC_IN_RC(dynamic));
4923 match(VecA);
4924 match(VecD);
4925 match(VecX);
4926
4927 op_cost(0);
4928 format %{ %}
4929 interface(REG_INTER);
4930 %}
4931
4932 operand vReg_V10()
4933 %{
4934 constraint(ALLOC_IN_RC(v10_veca_reg));
4935 match(vReg);
4936
4937 op_cost(0);
4938 format %{ %}
4939 interface(REG_INTER);
4940 %}
4941
4942 operand vReg_V11()
4943 %{
4944 constraint(ALLOC_IN_RC(v11_veca_reg));
4945 match(vReg);
4946
4947 op_cost(0);
4948 format %{ %}
4949 interface(REG_INTER);
4950 %}
4951
4952 operand vReg_V12()
4953 %{
4954 constraint(ALLOC_IN_RC(v12_veca_reg));
4955 match(vReg);
4956
4957 op_cost(0);
4958 format %{ %}
4959 interface(REG_INTER);
4960 %}
4961
4962 operand vReg_V13()
4963 %{
4964 constraint(ALLOC_IN_RC(v13_veca_reg));
4965 match(vReg);
4966
4967 op_cost(0);
4968 format %{ %}
4969 interface(REG_INTER);
4970 %}
4971
4972 operand vReg_V17()
4973 %{
4974 constraint(ALLOC_IN_RC(v17_veca_reg));
4975 match(vReg);
4976
4977 op_cost(0);
4978 format %{ %}
4979 interface(REG_INTER);
4980 %}
4981
4982 operand vReg_V18()
4983 %{
4984 constraint(ALLOC_IN_RC(v18_veca_reg));
4985 match(vReg);
4986
4987 op_cost(0);
4988 format %{ %}
4989 interface(REG_INTER);
4990 %}
4991
4992 operand vReg_V23()
4993 %{
4994 constraint(ALLOC_IN_RC(v23_veca_reg));
4995 match(vReg);
4996
4997 op_cost(0);
4998 format %{ %}
4999 interface(REG_INTER);
5000 %}
5001
5002 operand vReg_V24()
5003 %{
5004 constraint(ALLOC_IN_RC(v24_veca_reg));
5005 match(vReg);
5006
5007 op_cost(0);
5008 format %{ %}
5009 interface(REG_INTER);
5010 %}
5011
5012 operand vecA()
5013 %{
5014 constraint(ALLOC_IN_RC(vectora_reg));
5015 match(VecA);
5016
5017 op_cost(0);
5018 format %{ %}
5019 interface(REG_INTER);
5020 %}
5021
5022 operand vecD()
5023 %{
5024 constraint(ALLOC_IN_RC(vectord_reg));
5025 match(VecD);
5026
5027 op_cost(0);
5028 format %{ %}
5029 interface(REG_INTER);
5030 %}
5031
5032 operand vecX()
5033 %{
5034 constraint(ALLOC_IN_RC(vectorx_reg));
5035 match(VecX);
5036
5037 op_cost(0);
5038 format %{ %}
5039 interface(REG_INTER);
5040 %}
5041
5042 operand vRegD_V0()
5043 %{
5044 constraint(ALLOC_IN_RC(v0_reg));
5045 match(RegD);
5046 op_cost(0);
5047 format %{ %}
5048 interface(REG_INTER);
5049 %}
5050
5051 operand vRegD_V1()
5052 %{
5053 constraint(ALLOC_IN_RC(v1_reg));
5054 match(RegD);
5055 op_cost(0);
5056 format %{ %}
5057 interface(REG_INTER);
5058 %}
5059
5060 operand vRegD_V2()
5061 %{
5062 constraint(ALLOC_IN_RC(v2_reg));
5063 match(RegD);
5064 op_cost(0);
5065 format %{ %}
5066 interface(REG_INTER);
5067 %}
5068
5069 operand vRegD_V3()
5070 %{
5071 constraint(ALLOC_IN_RC(v3_reg));
5072 match(RegD);
5073 op_cost(0);
5074 format %{ %}
5075 interface(REG_INTER);
5076 %}
5077
5078 operand vRegD_V4()
5079 %{
5080 constraint(ALLOC_IN_RC(v4_reg));
5081 match(RegD);
5082 op_cost(0);
5083 format %{ %}
5084 interface(REG_INTER);
5085 %}
5086
5087 operand vRegD_V5()
5088 %{
5089 constraint(ALLOC_IN_RC(v5_reg));
5090 match(RegD);
5091 op_cost(0);
5092 format %{ %}
5093 interface(REG_INTER);
5094 %}
5095
5096 operand vRegD_V6()
5097 %{
5098 constraint(ALLOC_IN_RC(v6_reg));
5099 match(RegD);
5100 op_cost(0);
5101 format %{ %}
5102 interface(REG_INTER);
5103 %}
5104
5105 operand vRegD_V7()
5106 %{
5107 constraint(ALLOC_IN_RC(v7_reg));
5108 match(RegD);
5109 op_cost(0);
5110 format %{ %}
5111 interface(REG_INTER);
5112 %}
5113
5114 operand vRegD_V12()
5115 %{
5116 constraint(ALLOC_IN_RC(v12_reg));
5117 match(RegD);
5118 op_cost(0);
5119 format %{ %}
5120 interface(REG_INTER);
5121 %}
5122
5123 operand vRegD_V13()
5124 %{
5125 constraint(ALLOC_IN_RC(v13_reg));
5126 match(RegD);
5127 op_cost(0);
5128 format %{ %}
5129 interface(REG_INTER);
5130 %}
5131
5132 operand pReg()
5133 %{
5134 constraint(ALLOC_IN_RC(pr_reg));
5135 match(RegVectMask);
5136 match(pRegGov);
5137 op_cost(0);
5138 format %{ %}
5139 interface(REG_INTER);
5140 %}
5141
5142 operand pRegGov()
5143 %{
5144 constraint(ALLOC_IN_RC(gov_pr));
5145 match(RegVectMask);
5146 match(pReg);
5147 op_cost(0);
5148 format %{ %}
5149 interface(REG_INTER);
5150 %}
5151
5152 operand pRegGov_P0()
5153 %{
5154 constraint(ALLOC_IN_RC(p0_reg));
5155 match(RegVectMask);
5156 op_cost(0);
5157 format %{ %}
5158 interface(REG_INTER);
5159 %}
5160
5161 operand pRegGov_P1()
5162 %{
5163 constraint(ALLOC_IN_RC(p1_reg));
5164 match(RegVectMask);
5165 op_cost(0);
5166 format %{ %}
5167 interface(REG_INTER);
5168 %}
5169
5170 // Flags register, used as output of signed compare instructions
5171
5172 // note that on AArch64 we also use this register as the output for
5173 // for floating point compare instructions (CmpF CmpD). this ensures
5174 // that ordered inequality tests use GT, GE, LT or LE none of which
5175 // pass through cases where the result is unordered i.e. one or both
5176 // inputs to the compare is a NaN. this means that the ideal code can
5177 // replace e.g. a GT with an LE and not end up capturing the NaN case
5178 // (where the comparison should always fail). EQ and NE tests are
5179 // always generated in ideal code so that unordered folds into the NE
5180 // case, matching the behaviour of AArch64 NE.
5181 //
5182 // This differs from x86 where the outputs of FP compares use a
5183 // special FP flags registers and where compares based on this
5184 // register are distinguished into ordered inequalities (cmpOpUCF) and
5185 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5186 // to explicitly handle the unordered case in branches. x86 also has
5187 // to include extra CMoveX rules to accept a cmpOpUCF input.
5188
5189 operand rFlagsReg()
5190 %{
5191 constraint(ALLOC_IN_RC(int_flags));
5192 match(RegFlags);
5193
5194 op_cost(0);
5195 format %{ "RFLAGS" %}
5196 interface(REG_INTER);
5197 %}
5198
5199 // Flags register, used as output of unsigned compare instructions
5200 operand rFlagsRegU()
5201 %{
5202 constraint(ALLOC_IN_RC(int_flags));
5203 match(RegFlags);
5204
5205 op_cost(0);
5206 format %{ "RFLAGSU" %}
5207 interface(REG_INTER);
5208 %}
5209
5210 // Special Registers
5211
5212 // Method Register
5213 operand inline_cache_RegP(iRegP reg)
5214 %{
5215 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5216 match(reg);
5217 match(iRegPNoSp);
5218 op_cost(0);
5219 format %{ %}
5220 interface(REG_INTER);
5221 %}
5222
5223 // Thread Register
5224 operand thread_RegP(iRegP reg)
5225 %{
5226 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5227 match(reg);
5228 op_cost(0);
5229 format %{ %}
5230 interface(REG_INTER);
5231 %}
5232
5233 //----------Memory Operands----------------------------------------------------
5234
5235 operand indirect(iRegP reg)
5236 %{
5237 constraint(ALLOC_IN_RC(ptr_reg));
5238 match(reg);
5239 op_cost(0);
5240 format %{ "[$reg]" %}
5241 interface(MEMORY_INTER) %{
5242 base($reg);
5243 index(0xffffffff);
5244 scale(0x0);
5245 disp(0x0);
5246 %}
5247 %}
5248
5249 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5250 %{
5251 constraint(ALLOC_IN_RC(ptr_reg));
5252 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5253 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5254 op_cost(0);
5255 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5256 interface(MEMORY_INTER) %{
5257 base($reg);
5258 index($ireg);
5259 scale($scale);
5260 disp(0x0);
5261 %}
5262 %}
5263
5264 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5265 %{
5266 constraint(ALLOC_IN_RC(ptr_reg));
5267 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5268 match(AddP reg (LShiftL lreg scale));
5269 op_cost(0);
5270 format %{ "$reg, $lreg lsl($scale)" %}
5271 interface(MEMORY_INTER) %{
5272 base($reg);
5273 index($lreg);
5274 scale($scale);
5275 disp(0x0);
5276 %}
5277 %}
5278
5279 operand indIndexI2L(iRegP reg, iRegI ireg)
5280 %{
5281 constraint(ALLOC_IN_RC(ptr_reg));
5282 match(AddP reg (ConvI2L ireg));
5283 op_cost(0);
5284 format %{ "$reg, $ireg, 0, I2L" %}
5285 interface(MEMORY_INTER) %{
5286 base($reg);
5287 index($ireg);
5288 scale(0x0);
5289 disp(0x0);
5290 %}
5291 %}
5292
5293 operand indIndex(iRegP reg, iRegL lreg)
5294 %{
5295 constraint(ALLOC_IN_RC(ptr_reg));
5296 match(AddP reg lreg);
5297 op_cost(0);
5298 format %{ "$reg, $lreg" %}
5299 interface(MEMORY_INTER) %{
5300 base($reg);
5301 index($lreg);
5302 scale(0x0);
5303 disp(0x0);
5304 %}
5305 %}
5306
5307 operand indOffI1(iRegP reg, immIOffset1 off)
5308 %{
5309 constraint(ALLOC_IN_RC(ptr_reg));
5310 match(AddP reg off);
5311 op_cost(0);
5312 format %{ "[$reg, $off]" %}
5313 interface(MEMORY_INTER) %{
5314 base($reg);
5315 index(0xffffffff);
5316 scale(0x0);
5317 disp($off);
5318 %}
5319 %}
5320
5321 operand indOffI2(iRegP reg, immIOffset2 off)
5322 %{
5323 constraint(ALLOC_IN_RC(ptr_reg));
5324 match(AddP reg off);
5325 op_cost(0);
5326 format %{ "[$reg, $off]" %}
5327 interface(MEMORY_INTER) %{
5328 base($reg);
5329 index(0xffffffff);
5330 scale(0x0);
5331 disp($off);
5332 %}
5333 %}
5334
5335 operand indOffI4(iRegP reg, immIOffset4 off)
5336 %{
5337 constraint(ALLOC_IN_RC(ptr_reg));
5338 match(AddP reg off);
5339 op_cost(0);
5340 format %{ "[$reg, $off]" %}
5341 interface(MEMORY_INTER) %{
5342 base($reg);
5343 index(0xffffffff);
5344 scale(0x0);
5345 disp($off);
5346 %}
5347 %}
5348
5349 operand indOffI8(iRegP reg, immIOffset8 off)
5350 %{
5351 constraint(ALLOC_IN_RC(ptr_reg));
5352 match(AddP reg off);
5353 op_cost(0);
5354 format %{ "[$reg, $off]" %}
5355 interface(MEMORY_INTER) %{
5356 base($reg);
5357 index(0xffffffff);
5358 scale(0x0);
5359 disp($off);
5360 %}
5361 %}
5362
5363 operand indOffI16(iRegP reg, immIOffset16 off)
5364 %{
5365 constraint(ALLOC_IN_RC(ptr_reg));
5366 match(AddP reg off);
5367 op_cost(0);
5368 format %{ "[$reg, $off]" %}
5369 interface(MEMORY_INTER) %{
5370 base($reg);
5371 index(0xffffffff);
5372 scale(0x0);
5373 disp($off);
5374 %}
5375 %}
5376
5377 operand indOffL1(iRegP reg, immLoffset1 off)
5378 %{
5379 constraint(ALLOC_IN_RC(ptr_reg));
5380 match(AddP reg off);
5381 op_cost(0);
5382 format %{ "[$reg, $off]" %}
5383 interface(MEMORY_INTER) %{
5384 base($reg);
5385 index(0xffffffff);
5386 scale(0x0);
5387 disp($off);
5388 %}
5389 %}
5390
5391 operand indOffL2(iRegP reg, immLoffset2 off)
5392 %{
5393 constraint(ALLOC_IN_RC(ptr_reg));
5394 match(AddP reg off);
5395 op_cost(0);
5396 format %{ "[$reg, $off]" %}
5397 interface(MEMORY_INTER) %{
5398 base($reg);
5399 index(0xffffffff);
5400 scale(0x0);
5401 disp($off);
5402 %}
5403 %}
5404
5405 operand indOffL4(iRegP reg, immLoffset4 off)
5406 %{
5407 constraint(ALLOC_IN_RC(ptr_reg));
5408 match(AddP reg off);
5409 op_cost(0);
5410 format %{ "[$reg, $off]" %}
5411 interface(MEMORY_INTER) %{
5412 base($reg);
5413 index(0xffffffff);
5414 scale(0x0);
5415 disp($off);
5416 %}
5417 %}
5418
5419 operand indOffL8(iRegP reg, immLoffset8 off)
5420 %{
5421 constraint(ALLOC_IN_RC(ptr_reg));
5422 match(AddP reg off);
5423 op_cost(0);
5424 format %{ "[$reg, $off]" %}
5425 interface(MEMORY_INTER) %{
5426 base($reg);
5427 index(0xffffffff);
5428 scale(0x0);
5429 disp($off);
5430 %}
5431 %}
5432
5433 operand indOffL16(iRegP reg, immLoffset16 off)
5434 %{
5435 constraint(ALLOC_IN_RC(ptr_reg));
5436 match(AddP reg off);
5437 op_cost(0);
5438 format %{ "[$reg, $off]" %}
5439 interface(MEMORY_INTER) %{
5440 base($reg);
5441 index(0xffffffff);
5442 scale(0x0);
5443 disp($off);
5444 %}
5445 %}
5446
5447 operand indirectX2P(iRegL reg)
5448 %{
5449 constraint(ALLOC_IN_RC(ptr_reg));
5450 match(CastX2P reg);
5451 op_cost(0);
5452 format %{ "[$reg]\t# long -> ptr" %}
5453 interface(MEMORY_INTER) %{
5454 base($reg);
5455 index(0xffffffff);
5456 scale(0x0);
5457 disp(0x0);
5458 %}
5459 %}
5460
5461 operand indOffX2P(iRegL reg, immLOffset off)
5462 %{
5463 constraint(ALLOC_IN_RC(ptr_reg));
5464 match(AddP (CastX2P reg) off);
5465 op_cost(0);
5466 format %{ "[$reg, $off]\t# long -> ptr" %}
5467 interface(MEMORY_INTER) %{
5468 base($reg);
5469 index(0xffffffff);
5470 scale(0x0);
5471 disp($off);
5472 %}
5473 %}
5474
5475 operand indirectN(iRegN reg)
5476 %{
5477 predicate(CompressedOops::shift() == 0);
5478 constraint(ALLOC_IN_RC(ptr_reg));
5479 match(DecodeN reg);
5480 op_cost(0);
5481 format %{ "[$reg]\t# narrow" %}
5482 interface(MEMORY_INTER) %{
5483 base($reg);
5484 index(0xffffffff);
5485 scale(0x0);
5486 disp(0x0);
5487 %}
5488 %}
5489
5490 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5491 %{
5492 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5493 constraint(ALLOC_IN_RC(ptr_reg));
5494 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5495 op_cost(0);
5496 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5497 interface(MEMORY_INTER) %{
5498 base($reg);
5499 index($ireg);
5500 scale($scale);
5501 disp(0x0);
5502 %}
5503 %}
5504
5505 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5506 %{
5507 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5508 constraint(ALLOC_IN_RC(ptr_reg));
5509 match(AddP (DecodeN reg) (LShiftL lreg scale));
5510 op_cost(0);
5511 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5512 interface(MEMORY_INTER) %{
5513 base($reg);
5514 index($lreg);
5515 scale($scale);
5516 disp(0x0);
5517 %}
5518 %}
5519
5520 operand indIndexI2LN(iRegN reg, iRegI ireg)
5521 %{
5522 predicate(CompressedOops::shift() == 0);
5523 constraint(ALLOC_IN_RC(ptr_reg));
5524 match(AddP (DecodeN reg) (ConvI2L ireg));
5525 op_cost(0);
5526 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5527 interface(MEMORY_INTER) %{
5528 base($reg);
5529 index($ireg);
5530 scale(0x0);
5531 disp(0x0);
5532 %}
5533 %}
5534
5535 operand indIndexN(iRegN reg, iRegL lreg)
5536 %{
5537 predicate(CompressedOops::shift() == 0);
5538 constraint(ALLOC_IN_RC(ptr_reg));
5539 match(AddP (DecodeN reg) lreg);
5540 op_cost(0);
5541 format %{ "$reg, $lreg\t# narrow" %}
5542 interface(MEMORY_INTER) %{
5543 base($reg);
5544 index($lreg);
5545 scale(0x0);
5546 disp(0x0);
5547 %}
5548 %}
5549
5550 operand indOffIN(iRegN reg, immIOffset off)
5551 %{
5552 predicate(CompressedOops::shift() == 0);
5553 constraint(ALLOC_IN_RC(ptr_reg));
5554 match(AddP (DecodeN reg) off);
5555 op_cost(0);
5556 format %{ "[$reg, $off]\t# narrow" %}
5557 interface(MEMORY_INTER) %{
5558 base($reg);
5559 index(0xffffffff);
5560 scale(0x0);
5561 disp($off);
5562 %}
5563 %}
5564
5565 operand indOffLN(iRegN reg, immLOffset off)
5566 %{
5567 predicate(CompressedOops::shift() == 0);
5568 constraint(ALLOC_IN_RC(ptr_reg));
5569 match(AddP (DecodeN reg) off);
5570 op_cost(0);
5571 format %{ "[$reg, $off]\t# narrow" %}
5572 interface(MEMORY_INTER) %{
5573 base($reg);
5574 index(0xffffffff);
5575 scale(0x0);
5576 disp($off);
5577 %}
5578 %}
5579
5580
5581 //----------Special Memory Operands--------------------------------------------
5582 // Stack Slot Operand - This operand is used for loading and storing temporary
5583 // values on the stack where a match requires a value to
5584 // flow through memory.
5585 operand stackSlotP(sRegP reg)
5586 %{
5587 constraint(ALLOC_IN_RC(stack_slots));
5588 op_cost(100);
5589 // No match rule because this operand is only generated in matching
5590 // match(RegP);
5591 format %{ "[$reg]" %}
5592 interface(MEMORY_INTER) %{
5593 base(0x1e); // RSP
5594 index(0x0); // No Index
5595 scale(0x0); // No Scale
5596 disp($reg); // Stack Offset
5597 %}
5598 %}
5599
5600 operand stackSlotI(sRegI reg)
5601 %{
5602 constraint(ALLOC_IN_RC(stack_slots));
5603 // No match rule because this operand is only generated in matching
5604 // match(RegI);
5605 format %{ "[$reg]" %}
5606 interface(MEMORY_INTER) %{
5607 base(0x1e); // RSP
5608 index(0x0); // No Index
5609 scale(0x0); // No Scale
5610 disp($reg); // Stack Offset
5611 %}
5612 %}
5613
5614 operand stackSlotF(sRegF reg)
5615 %{
5616 constraint(ALLOC_IN_RC(stack_slots));
5617 // No match rule because this operand is only generated in matching
5618 // match(RegF);
5619 format %{ "[$reg]" %}
5620 interface(MEMORY_INTER) %{
5621 base(0x1e); // RSP
5622 index(0x0); // No Index
5623 scale(0x0); // No Scale
5624 disp($reg); // Stack Offset
5625 %}
5626 %}
5627
5628 operand stackSlotD(sRegD reg)
5629 %{
5630 constraint(ALLOC_IN_RC(stack_slots));
5631 // No match rule because this operand is only generated in matching
5632 // match(RegD);
5633 format %{ "[$reg]" %}
5634 interface(MEMORY_INTER) %{
5635 base(0x1e); // RSP
5636 index(0x0); // No Index
5637 scale(0x0); // No Scale
5638 disp($reg); // Stack Offset
5639 %}
5640 %}
5641
5642 operand stackSlotL(sRegL reg)
5643 %{
5644 constraint(ALLOC_IN_RC(stack_slots));
5645 // No match rule because this operand is only generated in matching
5646 // match(RegL);
5647 format %{ "[$reg]" %}
5648 interface(MEMORY_INTER) %{
5649 base(0x1e); // RSP
5650 index(0x0); // No Index
5651 scale(0x0); // No Scale
5652 disp($reg); // Stack Offset
5653 %}
5654 %}
5655
5656 // Operands for expressing Control Flow
5657 // NOTE: Label is a predefined operand which should not be redefined in
5658 // the AD file. It is generically handled within the ADLC.
5659
5660 //----------Conditional Branch Operands----------------------------------------
5661 // Comparison Op - This is the operation of the comparison, and is limited to
5662 // the following set of codes:
5663 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5664 //
5665 // Other attributes of the comparison, such as unsignedness, are specified
5666 // by the comparison instruction that sets a condition code flags register.
5667 // That result is represented by a flags operand whose subtype is appropriate
5668 // to the unsignedness (etc.) of the comparison.
5669 //
5670 // Later, the instruction which matches both the Comparison Op (a Bool) and
5671 // the flags (produced by the Cmp) specifies the coding of the comparison op
5672 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5673
5674 // used for signed integral comparisons and fp comparisons
5675
5676 operand cmpOp()
5677 %{
5678 match(Bool);
5679
5680 format %{ "" %}
5681 interface(COND_INTER) %{
5682 equal(0x0, "eq");
5683 not_equal(0x1, "ne");
5684 less(0xb, "lt");
5685 greater_equal(0xa, "ge");
5686 less_equal(0xd, "le");
5687 greater(0xc, "gt");
5688 overflow(0x6, "vs");
5689 no_overflow(0x7, "vc");
5690 %}
5691 %}
5692
5693 // used for unsigned integral comparisons
5694
5695 operand cmpOpU()
5696 %{
5697 match(Bool);
5698
5699 format %{ "" %}
5700 interface(COND_INTER) %{
5701 equal(0x0, "eq");
5702 not_equal(0x1, "ne");
5703 less(0x3, "lo");
5704 greater_equal(0x2, "hs");
5705 less_equal(0x9, "ls");
5706 greater(0x8, "hi");
5707 overflow(0x6, "vs");
5708 no_overflow(0x7, "vc");
5709 %}
5710 %}
5711
5712 // used for certain integral comparisons which can be
5713 // converted to cbxx or tbxx instructions
5714
5715 operand cmpOpEqNe()
5716 %{
5717 match(Bool);
5718 op_cost(0);
5719 predicate(n->as_Bool()->_test._test == BoolTest::ne
5720 || n->as_Bool()->_test._test == BoolTest::eq);
5721
5722 format %{ "" %}
5723 interface(COND_INTER) %{
5724 equal(0x0, "eq");
5725 not_equal(0x1, "ne");
5726 less(0xb, "lt");
5727 greater_equal(0xa, "ge");
5728 less_equal(0xd, "le");
5729 greater(0xc, "gt");
5730 overflow(0x6, "vs");
5731 no_overflow(0x7, "vc");
5732 %}
5733 %}
5734
5735 // used for certain integral comparisons which can be
5736 // converted to cbxx or tbxx instructions
5737
5738 operand cmpOpLtGe()
5739 %{
5740 match(Bool);
5741 op_cost(0);
5742
5743 predicate(n->as_Bool()->_test._test == BoolTest::lt
5744 || n->as_Bool()->_test._test == BoolTest::ge);
5745
5746 format %{ "" %}
5747 interface(COND_INTER) %{
5748 equal(0x0, "eq");
5749 not_equal(0x1, "ne");
5750 less(0xb, "lt");
5751 greater_equal(0xa, "ge");
5752 less_equal(0xd, "le");
5753 greater(0xc, "gt");
5754 overflow(0x6, "vs");
5755 no_overflow(0x7, "vc");
5756 %}
5757 %}
5758
5759 // used for certain unsigned integral comparisons which can be
5760 // converted to cbxx or tbxx instructions
5761
5762 operand cmpOpUEqNeLeGt()
5763 %{
5764 match(Bool);
5765 op_cost(0);
5766
5767 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5768 n->as_Bool()->_test._test == BoolTest::ne ||
5769 n->as_Bool()->_test._test == BoolTest::le ||
5770 n->as_Bool()->_test._test == BoolTest::gt);
5771
5772 format %{ "" %}
5773 interface(COND_INTER) %{
5774 equal(0x0, "eq");
5775 not_equal(0x1, "ne");
5776 less(0x3, "lo");
5777 greater_equal(0x2, "hs");
5778 less_equal(0x9, "ls");
5779 greater(0x8, "hi");
5780 overflow(0x6, "vs");
5781 no_overflow(0x7, "vc");
5782 %}
5783 %}
5784
5785 // Special operand allowing long args to int ops to be truncated for free
5786
5787 operand iRegL2I(iRegL reg) %{
5788
5789 op_cost(0);
5790
5791 match(ConvL2I reg);
5792
5793 format %{ "l2i($reg)" %}
5794
5795 interface(REG_INTER)
5796 %}
5797
5798 operand iRegL2P(iRegL reg) %{
5799
5800 op_cost(0);
5801
5802 match(CastX2P reg);
5803
5804 format %{ "l2p($reg)" %}
5805
5806 interface(REG_INTER)
5807 %}
5808
5809 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5810 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5811 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5812 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5813
5814 //----------OPERAND CLASSES----------------------------------------------------
5815 // Operand Classes are groups of operands that are used as to simplify
5816 // instruction definitions by not requiring the AD writer to specify
5817 // separate instructions for every form of operand when the
5818 // instruction accepts multiple operand types with the same basic
5819 // encoding and format. The classic case of this is memory operands.
5820
5821 // memory is used to define read/write location for load/store
5822 // instruction defs. we can turn a memory op into an Address
5823
5824 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5825 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5826
5827 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5828 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5829
5830 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5831 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5832
5833 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5834 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5835
5836 // All of the memory operands. For the pipeline description.
5837 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5838 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5839 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5840
5841
5842 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5843 // operations. it allows the src to be either an iRegI or a (ConvL2I
5844 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5845 // can be elided because the 32-bit instruction will just employ the
5846 // lower 32 bits anyway.
5847 //
5848 // n.b. this does not elide all L2I conversions. if the truncated
5849 // value is consumed by more than one operation then the ConvL2I
5850 // cannot be bundled into the consuming nodes so an l2i gets planted
5851 // (actually a movw $dst $src) and the downstream instructions consume
5852 // the result of the l2i as an iRegI input. That's a shame since the
5853 // movw is actually redundant but its not too costly.
5854
5855 opclass iRegIorL2I(iRegI, iRegL2I);
5856 opclass iRegPorL2P(iRegP, iRegL2P);
5857
5858 //----------PIPELINE-----------------------------------------------------------
5859 // Rules which define the behavior of the target architectures pipeline.
5860
5861 // For specific pipelines, eg A53, define the stages of that pipeline
5862 //pipe_desc(ISS, EX1, EX2, WR);
5863 #define ISS S0
5864 #define EX1 S1
5865 #define EX2 S2
5866 #define WR S3
5867
5868 // Integer ALU reg operation
5869 pipeline %{
5870
5871 attributes %{
5872 // ARM instructions are of fixed length
5873 fixed_size_instructions; // Fixed size instructions TODO does
5874 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5875 // ARM instructions come in 32-bit word units
5876 instruction_unit_size = 4; // An instruction is 4 bytes long
5877 instruction_fetch_unit_size = 64; // The processor fetches one line
5878 instruction_fetch_units = 1; // of 64 bytes
5879 %}
5880
5881 // We don't use an actual pipeline model so don't care about resources
5882 // or description. we do use pipeline classes to introduce fixed
5883 // latencies
5884
5885 //----------RESOURCES----------------------------------------------------------
5886 // Resources are the functional units available to the machine
5887
5888 resources( INS0, INS1, INS01 = INS0 | INS1,
5889 ALU0, ALU1, ALU = ALU0 | ALU1,
5890 MAC,
5891 DIV,
5892 BRANCH,
5893 LDST,
5894 NEON_FP);
5895
5896 //----------PIPELINE DESCRIPTION-----------------------------------------------
5897 // Pipeline Description specifies the stages in the machine's pipeline
5898
5899 // Define the pipeline as a generic 6 stage pipeline
5900 pipe_desc(S0, S1, S2, S3, S4, S5);
5901
5902 //----------PIPELINE CLASSES---------------------------------------------------
5903 // Pipeline Classes describe the stages in which input and output are
5904 // referenced by the hardware pipeline.
5905
5906 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5907 %{
5908 single_instruction;
5909 src1 : S1(read);
5910 src2 : S2(read);
5911 dst : S5(write);
5912 INS01 : ISS;
5913 NEON_FP : S5;
5914 %}
5915
5916 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5917 %{
5918 single_instruction;
5919 src1 : S1(read);
5920 src2 : S2(read);
5921 dst : S5(write);
5922 INS01 : ISS;
5923 NEON_FP : S5;
5924 %}
5925
5926 pipe_class fp_uop_s(vRegF dst, vRegF src)
5927 %{
5928 single_instruction;
5929 src : S1(read);
5930 dst : S5(write);
5931 INS01 : ISS;
5932 NEON_FP : S5;
5933 %}
5934
5935 pipe_class fp_uop_d(vRegD dst, vRegD src)
5936 %{
5937 single_instruction;
5938 src : S1(read);
5939 dst : S5(write);
5940 INS01 : ISS;
5941 NEON_FP : S5;
5942 %}
5943
5944 pipe_class fp_d2f(vRegF dst, vRegD src)
5945 %{
5946 single_instruction;
5947 src : S1(read);
5948 dst : S5(write);
5949 INS01 : ISS;
5950 NEON_FP : S5;
5951 %}
5952
5953 pipe_class fp_f2d(vRegD dst, vRegF src)
5954 %{
5955 single_instruction;
5956 src : S1(read);
5957 dst : S5(write);
5958 INS01 : ISS;
5959 NEON_FP : S5;
5960 %}
5961
5962 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5963 %{
5964 single_instruction;
5965 src : S1(read);
5966 dst : S5(write);
5967 INS01 : ISS;
5968 NEON_FP : S5;
5969 %}
5970
5971 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5972 %{
5973 single_instruction;
5974 src : S1(read);
5975 dst : S5(write);
5976 INS01 : ISS;
5977 NEON_FP : S5;
5978 %}
5979
5980 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5981 %{
5982 single_instruction;
5983 src : S1(read);
5984 dst : S5(write);
5985 INS01 : ISS;
5986 NEON_FP : S5;
5987 %}
5988
5989 pipe_class fp_l2f(vRegF dst, iRegL src)
5990 %{
5991 single_instruction;
5992 src : S1(read);
5993 dst : S5(write);
5994 INS01 : ISS;
5995 NEON_FP : S5;
5996 %}
5997
5998 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5999 %{
6000 single_instruction;
6001 src : S1(read);
6002 dst : S5(write);
6003 INS01 : ISS;
6004 NEON_FP : S5;
6005 %}
6006
6007 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6008 %{
6009 single_instruction;
6010 src : S1(read);
6011 dst : S5(write);
6012 INS01 : ISS;
6013 NEON_FP : S5;
6014 %}
6015
6016 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6017 %{
6018 single_instruction;
6019 src : S1(read);
6020 dst : S5(write);
6021 INS01 : ISS;
6022 NEON_FP : S5;
6023 %}
6024
6025 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6026 %{
6027 single_instruction;
6028 src : S1(read);
6029 dst : S5(write);
6030 INS01 : ISS;
6031 NEON_FP : S5;
6032 %}
6033
6034 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6035 %{
6036 single_instruction;
6037 src1 : S1(read);
6038 src2 : S2(read);
6039 dst : S5(write);
6040 INS0 : ISS;
6041 NEON_FP : S5;
6042 %}
6043
6044 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6045 %{
6046 single_instruction;
6047 src1 : S1(read);
6048 src2 : S2(read);
6049 dst : S5(write);
6050 INS0 : ISS;
6051 NEON_FP : S5;
6052 %}
6053
6054 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6055 %{
6056 single_instruction;
6057 cr : S1(read);
6058 src1 : S1(read);
6059 src2 : S1(read);
6060 dst : S3(write);
6061 INS01 : ISS;
6062 NEON_FP : S3;
6063 %}
6064
6065 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6066 %{
6067 single_instruction;
6068 cr : S1(read);
6069 src1 : S1(read);
6070 src2 : S1(read);
6071 dst : S3(write);
6072 INS01 : ISS;
6073 NEON_FP : S3;
6074 %}
6075
6076 pipe_class fp_imm_s(vRegF dst)
6077 %{
6078 single_instruction;
6079 dst : S3(write);
6080 INS01 : ISS;
6081 NEON_FP : S3;
6082 %}
6083
6084 pipe_class fp_imm_d(vRegD dst)
6085 %{
6086 single_instruction;
6087 dst : S3(write);
6088 INS01 : ISS;
6089 NEON_FP : S3;
6090 %}
6091
6092 pipe_class fp_load_constant_s(vRegF dst)
6093 %{
6094 single_instruction;
6095 dst : S4(write);
6096 INS01 : ISS;
6097 NEON_FP : S4;
6098 %}
6099
6100 pipe_class fp_load_constant_d(vRegD dst)
6101 %{
6102 single_instruction;
6103 dst : S4(write);
6104 INS01 : ISS;
6105 NEON_FP : S4;
6106 %}
6107
6108 //------- Integer ALU operations --------------------------
6109
6110 // Integer ALU reg-reg operation
6111 // Operands needed in EX1, result generated in EX2
6112 // Eg. ADD x0, x1, x2
6113 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6114 %{
6115 single_instruction;
6116 dst : EX2(write);
6117 src1 : EX1(read);
6118 src2 : EX1(read);
6119 INS01 : ISS; // Dual issue as instruction 0 or 1
6120 ALU : EX2;
6121 %}
6122
6123 // Integer ALU reg-reg operation with constant shift
6124 // Shifted register must be available in LATE_ISS instead of EX1
6125 // Eg. ADD x0, x1, x2, LSL #2
6126 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6127 %{
6128 single_instruction;
6129 dst : EX2(write);
6130 src1 : EX1(read);
6131 src2 : ISS(read);
6132 INS01 : ISS;
6133 ALU : EX2;
6134 %}
6135
6136 // Integer ALU reg operation with constant shift
6137 // Eg. LSL x0, x1, #shift
6138 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6139 %{
6140 single_instruction;
6141 dst : EX2(write);
6142 src1 : ISS(read);
6143 INS01 : ISS;
6144 ALU : EX2;
6145 %}
6146
6147 // Integer ALU reg-reg operation with variable shift
6148 // Both operands must be available in LATE_ISS instead of EX1
6149 // Result is available in EX1 instead of EX2
6150 // Eg. LSLV x0, x1, x2
6151 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6152 %{
6153 single_instruction;
6154 dst : EX1(write);
6155 src1 : ISS(read);
6156 src2 : ISS(read);
6157 INS01 : ISS;
6158 ALU : EX1;
6159 %}
6160
6161 // Integer ALU reg-reg operation with extract
6162 // As for _vshift above, but result generated in EX2
6163 // Eg. EXTR x0, x1, x2, #N
6164 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6165 %{
6166 single_instruction;
6167 dst : EX2(write);
6168 src1 : ISS(read);
6169 src2 : ISS(read);
6170 INS1 : ISS; // Can only dual issue as Instruction 1
6171 ALU : EX1;
6172 %}
6173
6174 // Integer ALU reg operation
6175 // Eg. NEG x0, x1
6176 pipe_class ialu_reg(iRegI dst, iRegI src)
6177 %{
6178 single_instruction;
6179 dst : EX2(write);
6180 src : EX1(read);
6181 INS01 : ISS;
6182 ALU : EX2;
6183 %}
6184
6185 // Integer ALU reg mmediate operation
6186 // Eg. ADD x0, x1, #N
6187 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6188 %{
6189 single_instruction;
6190 dst : EX2(write);
6191 src1 : EX1(read);
6192 INS01 : ISS;
6193 ALU : EX2;
6194 %}
6195
6196 // Integer ALU immediate operation (no source operands)
6197 // Eg. MOV x0, #N
6198 pipe_class ialu_imm(iRegI dst)
6199 %{
6200 single_instruction;
6201 dst : EX1(write);
6202 INS01 : ISS;
6203 ALU : EX1;
6204 %}
6205
6206 //------- Compare operation -------------------------------
6207
6208 // Compare reg-reg
6209 // Eg. CMP x0, x1
6210 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6211 %{
6212 single_instruction;
6213 // fixed_latency(16);
6214 cr : EX2(write);
6215 op1 : EX1(read);
6216 op2 : EX1(read);
6217 INS01 : ISS;
6218 ALU : EX2;
6219 %}
6220
6221 // Compare reg-reg
6222 // Eg. CMP x0, #N
6223 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6224 %{
6225 single_instruction;
6226 // fixed_latency(16);
6227 cr : EX2(write);
6228 op1 : EX1(read);
6229 INS01 : ISS;
6230 ALU : EX2;
6231 %}
6232
6233 //------- Conditional instructions ------------------------
6234
6235 // Conditional no operands
6236 // Eg. CSINC x0, zr, zr, <cond>
6237 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6238 %{
6239 single_instruction;
6240 cr : EX1(read);
6241 dst : EX2(write);
6242 INS01 : ISS;
6243 ALU : EX2;
6244 %}
6245
6246 // Conditional 2 operand
6247 // EG. CSEL X0, X1, X2, <cond>
6248 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6249 %{
6250 single_instruction;
6251 cr : EX1(read);
6252 src1 : EX1(read);
6253 src2 : EX1(read);
6254 dst : EX2(write);
6255 INS01 : ISS;
6256 ALU : EX2;
6257 %}
6258
6259 // Conditional 2 operand
6260 // EG. CSEL X0, X1, X2, <cond>
6261 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6262 %{
6263 single_instruction;
6264 cr : EX1(read);
6265 src : EX1(read);
6266 dst : EX2(write);
6267 INS01 : ISS;
6268 ALU : EX2;
6269 %}
6270
6271 //------- Multiply pipeline operations --------------------
6272
6273 // Multiply reg-reg
6274 // Eg. MUL w0, w1, w2
6275 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6276 %{
6277 single_instruction;
6278 dst : WR(write);
6279 src1 : ISS(read);
6280 src2 : ISS(read);
6281 INS01 : ISS;
6282 MAC : WR;
6283 %}
6284
6285 // Multiply accumulate
6286 // Eg. MADD w0, w1, w2, w3
6287 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6288 %{
6289 single_instruction;
6290 dst : WR(write);
6291 src1 : ISS(read);
6292 src2 : ISS(read);
6293 src3 : ISS(read);
6294 INS01 : ISS;
6295 MAC : WR;
6296 %}
6297
6298 // Eg. MUL w0, w1, w2
6299 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6300 %{
6301 single_instruction;
6302 fixed_latency(3); // Maximum latency for 64 bit mul
6303 dst : WR(write);
6304 src1 : ISS(read);
6305 src2 : ISS(read);
6306 INS01 : ISS;
6307 MAC : WR;
6308 %}
6309
6310 // Multiply accumulate
6311 // Eg. MADD w0, w1, w2, w3
6312 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6313 %{
6314 single_instruction;
6315 fixed_latency(3); // Maximum latency for 64 bit mul
6316 dst : WR(write);
6317 src1 : ISS(read);
6318 src2 : ISS(read);
6319 src3 : ISS(read);
6320 INS01 : ISS;
6321 MAC : WR;
6322 %}
6323
6324 //------- Divide pipeline operations --------------------
6325
6326 // Eg. SDIV w0, w1, w2
6327 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6328 %{
6329 single_instruction;
6330 fixed_latency(8); // Maximum latency for 32 bit divide
6331 dst : WR(write);
6332 src1 : ISS(read);
6333 src2 : ISS(read);
6334 INS0 : ISS; // Can only dual issue as instruction 0
6335 DIV : WR;
6336 %}
6337
6338 // Eg. SDIV x0, x1, x2
6339 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6340 %{
6341 single_instruction;
6342 fixed_latency(16); // Maximum latency for 64 bit divide
6343 dst : WR(write);
6344 src1 : ISS(read);
6345 src2 : ISS(read);
6346 INS0 : ISS; // Can only dual issue as instruction 0
6347 DIV : WR;
6348 %}
6349
6350 //------- Load pipeline operations ------------------------
6351
6352 // Load - prefetch
6353 // Eg. PFRM <mem>
6354 pipe_class iload_prefetch(memory mem)
6355 %{
6356 single_instruction;
6357 mem : ISS(read);
6358 INS01 : ISS;
6359 LDST : WR;
6360 %}
6361
6362 // Load - reg, mem
6363 // Eg. LDR x0, <mem>
6364 pipe_class iload_reg_mem(iRegI dst, memory mem)
6365 %{
6366 single_instruction;
6367 dst : WR(write);
6368 mem : ISS(read);
6369 INS01 : ISS;
6370 LDST : WR;
6371 %}
6372
6373 // Load - reg, reg
6374 // Eg. LDR x0, [sp, x1]
6375 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6376 %{
6377 single_instruction;
6378 dst : WR(write);
6379 src : ISS(read);
6380 INS01 : ISS;
6381 LDST : WR;
6382 %}
6383
6384 //------- Store pipeline operations -----------------------
6385
6386 // Store - zr, mem
6387 // Eg. STR zr, <mem>
6388 pipe_class istore_mem(memory mem)
6389 %{
6390 single_instruction;
6391 mem : ISS(read);
6392 INS01 : ISS;
6393 LDST : WR;
6394 %}
6395
6396 // Store - reg, mem
6397 // Eg. STR x0, <mem>
6398 pipe_class istore_reg_mem(iRegI src, memory mem)
6399 %{
6400 single_instruction;
6401 mem : ISS(read);
6402 src : EX2(read);
6403 INS01 : ISS;
6404 LDST : WR;
6405 %}
6406
6407 // Store - reg, reg
6408 // Eg. STR x0, [sp, x1]
6409 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6410 %{
6411 single_instruction;
6412 dst : ISS(read);
6413 src : EX2(read);
6414 INS01 : ISS;
6415 LDST : WR;
6416 %}
6417
6418 //------- Store pipeline operations -----------------------
6419
6420 // Branch
6421 pipe_class pipe_branch()
6422 %{
6423 single_instruction;
6424 INS01 : ISS;
6425 BRANCH : EX1;
6426 %}
6427
6428 // Conditional branch
6429 pipe_class pipe_branch_cond(rFlagsReg cr)
6430 %{
6431 single_instruction;
6432 cr : EX1(read);
6433 INS01 : ISS;
6434 BRANCH : EX1;
6435 %}
6436
6437 // Compare & Branch
6438 // EG. CBZ/CBNZ
6439 pipe_class pipe_cmp_branch(iRegI op1)
6440 %{
6441 single_instruction;
6442 op1 : EX1(read);
6443 INS01 : ISS;
6444 BRANCH : EX1;
6445 %}
6446
6447 //------- Synchronisation operations ----------------------
6448
6449 // Any operation requiring serialization.
6450 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6451 pipe_class pipe_serial()
6452 %{
6453 single_instruction;
6454 force_serialization;
6455 fixed_latency(16);
6456 INS01 : ISS(2); // Cannot dual issue with any other instruction
6457 LDST : WR;
6458 %}
6459
6460 // Generic big/slow expanded idiom - also serialized
6461 pipe_class pipe_slow()
6462 %{
6463 instruction_count(10);
6464 multiple_bundles;
6465 force_serialization;
6466 fixed_latency(16);
6467 INS01 : ISS(2); // Cannot dual issue with any other instruction
6468 LDST : WR;
6469 %}
6470
6471 // Empty pipeline class
6472 pipe_class pipe_class_empty()
6473 %{
6474 single_instruction;
6475 fixed_latency(0);
6476 %}
6477
6478 // Default pipeline class.
6479 pipe_class pipe_class_default()
6480 %{
6481 single_instruction;
6482 fixed_latency(2);
6483 %}
6484
6485 // Pipeline class for compares.
6486 pipe_class pipe_class_compare()
6487 %{
6488 single_instruction;
6489 fixed_latency(16);
6490 %}
6491
6492 // Pipeline class for memory operations.
6493 pipe_class pipe_class_memory()
6494 %{
6495 single_instruction;
6496 fixed_latency(16);
6497 %}
6498
6499 // Pipeline class for call.
6500 pipe_class pipe_class_call()
6501 %{
6502 single_instruction;
6503 fixed_latency(100);
6504 %}
6505
6506 // Define the class for the Nop node.
6507 define %{
6508 MachNop = pipe_class_empty;
6509 %}
6510
6511 %}
6512 //----------INSTRUCTIONS-------------------------------------------------------
6513 //
6514 // match -- States which machine-independent subtree may be replaced
6515 // by this instruction.
6516 // ins_cost -- The estimated cost of this instruction is used by instruction
6517 // selection to identify a minimum cost tree of machine
6518 // instructions that matches a tree of machine-independent
6519 // instructions.
6520 // format -- A string providing the disassembly for this instruction.
6521 // The value of an instruction's operand may be inserted
6522 // by referring to it with a '$' prefix.
6523 // opcode -- Three instruction opcodes may be provided. These are referred
6524 // to within an encode class as $primary, $secondary, and $tertiary
6525 // rrspectively. The primary opcode is commonly used to
6526 // indicate the type of machine instruction, while secondary
6527 // and tertiary are often used for prefix options or addressing
6528 // modes.
6529 // ins_encode -- A list of encode classes with parameters. The encode class
6530 // name must have been defined in an 'enc_class' specification
6531 // in the encode section of the architecture description.
6532
6533 // ============================================================================
6534 // Memory (Load/Store) Instructions
6535
6536 // Load Instructions
6537
6538 // Load Byte (8 bit signed)
6539 instruct loadB(iRegINoSp dst, memory1 mem)
6540 %{
6541 match(Set dst (LoadB mem));
6542 predicate(!needs_acquiring_load(n));
6543
6544 ins_cost(4 * INSN_COST);
6545 format %{ "ldrsbw $dst, $mem\t# byte" %}
6546
6547 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6548
6549 ins_pipe(iload_reg_mem);
6550 %}
6551
6552 // Load Byte (8 bit signed) into long
6553 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6554 %{
6555 match(Set dst (ConvI2L (LoadB mem)));
6556 predicate(!needs_acquiring_load(n->in(1)));
6557
6558 ins_cost(4 * INSN_COST);
6559 format %{ "ldrsb $dst, $mem\t# byte" %}
6560
6561 ins_encode(aarch64_enc_ldrsb(dst, mem));
6562
6563 ins_pipe(iload_reg_mem);
6564 %}
6565
6566 // Load Byte (8 bit unsigned)
6567 instruct loadUB(iRegINoSp dst, memory1 mem)
6568 %{
6569 match(Set dst (LoadUB mem));
6570 predicate(!needs_acquiring_load(n));
6571
6572 ins_cost(4 * INSN_COST);
6573 format %{ "ldrbw $dst, $mem\t# byte" %}
6574
6575 ins_encode(aarch64_enc_ldrb(dst, mem));
6576
6577 ins_pipe(iload_reg_mem);
6578 %}
6579
6580 // Load Byte (8 bit unsigned) into long
6581 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6582 %{
6583 match(Set dst (ConvI2L (LoadUB mem)));
6584 predicate(!needs_acquiring_load(n->in(1)));
6585
6586 ins_cost(4 * INSN_COST);
6587 format %{ "ldrb $dst, $mem\t# byte" %}
6588
6589 ins_encode(aarch64_enc_ldrb(dst, mem));
6590
6591 ins_pipe(iload_reg_mem);
6592 %}
6593
6594 // Load Short (16 bit signed)
6595 instruct loadS(iRegINoSp dst, memory2 mem)
6596 %{
6597 match(Set dst (LoadS mem));
6598 predicate(!needs_acquiring_load(n));
6599
6600 ins_cost(4 * INSN_COST);
6601 format %{ "ldrshw $dst, $mem\t# short" %}
6602
6603 ins_encode(aarch64_enc_ldrshw(dst, mem));
6604
6605 ins_pipe(iload_reg_mem);
6606 %}
6607
6608 // Load Short (16 bit signed) into long
6609 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6610 %{
6611 match(Set dst (ConvI2L (LoadS mem)));
6612 predicate(!needs_acquiring_load(n->in(1)));
6613
6614 ins_cost(4 * INSN_COST);
6615 format %{ "ldrsh $dst, $mem\t# short" %}
6616
6617 ins_encode(aarch64_enc_ldrsh(dst, mem));
6618
6619 ins_pipe(iload_reg_mem);
6620 %}
6621
6622 // Load Char (16 bit unsigned)
6623 instruct loadUS(iRegINoSp dst, memory2 mem)
6624 %{
6625 match(Set dst (LoadUS mem));
6626 predicate(!needs_acquiring_load(n));
6627
6628 ins_cost(4 * INSN_COST);
6629 format %{ "ldrh $dst, $mem\t# short" %}
6630
6631 ins_encode(aarch64_enc_ldrh(dst, mem));
6632
6633 ins_pipe(iload_reg_mem);
6634 %}
6635
6636 // Load Short/Char (16 bit unsigned) into long
6637 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6638 %{
6639 match(Set dst (ConvI2L (LoadUS mem)));
6640 predicate(!needs_acquiring_load(n->in(1)));
6641
6642 ins_cost(4 * INSN_COST);
6643 format %{ "ldrh $dst, $mem\t# short" %}
6644
6645 ins_encode(aarch64_enc_ldrh(dst, mem));
6646
6647 ins_pipe(iload_reg_mem);
6648 %}
6649
6650 // Load Integer (32 bit signed)
6651 instruct loadI(iRegINoSp dst, memory4 mem)
6652 %{
6653 match(Set dst (LoadI mem));
6654 predicate(!needs_acquiring_load(n));
6655
6656 ins_cost(4 * INSN_COST);
6657 format %{ "ldrw $dst, $mem\t# int" %}
6658
6659 ins_encode(aarch64_enc_ldrw(dst, mem));
6660
6661 ins_pipe(iload_reg_mem);
6662 %}
6663
6664 // Load Integer (32 bit signed) into long
6665 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6666 %{
6667 match(Set dst (ConvI2L (LoadI mem)));
6668 predicate(!needs_acquiring_load(n->in(1)));
6669
6670 ins_cost(4 * INSN_COST);
6671 format %{ "ldrsw $dst, $mem\t# int" %}
6672
6673 ins_encode(aarch64_enc_ldrsw(dst, mem));
6674
6675 ins_pipe(iload_reg_mem);
6676 %}
6677
6678 // Load Integer (32 bit unsigned) into long
6679 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6680 %{
6681 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6682 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6683
6684 ins_cost(4 * INSN_COST);
6685 format %{ "ldrw $dst, $mem\t# int" %}
6686
6687 ins_encode(aarch64_enc_ldrw(dst, mem));
6688
6689 ins_pipe(iload_reg_mem);
6690 %}
6691
6692 // Load Long (64 bit signed)
6693 instruct loadL(iRegLNoSp dst, memory8 mem)
6694 %{
6695 match(Set dst (LoadL mem));
6696 predicate(!needs_acquiring_load(n));
6697
6698 ins_cost(4 * INSN_COST);
6699 format %{ "ldr $dst, $mem\t# int" %}
6700
6701 ins_encode(aarch64_enc_ldr(dst, mem));
6702
6703 ins_pipe(iload_reg_mem);
6704 %}
6705
6706 // Load Range
6707 instruct loadRange(iRegINoSp dst, memory4 mem)
6708 %{
6709 match(Set dst (LoadRange mem));
6710
6711 ins_cost(4 * INSN_COST);
6712 format %{ "ldrw $dst, $mem\t# range" %}
6713
6714 ins_encode(aarch64_enc_ldrw(dst, mem));
6715
6716 ins_pipe(iload_reg_mem);
6717 %}
6718
6719 // Load Pointer
6720 instruct loadP(iRegPNoSp dst, memory8 mem)
6721 %{
6722 match(Set dst (LoadP mem));
6723 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6724
6725 ins_cost(4 * INSN_COST);
6726 format %{ "ldr $dst, $mem\t# ptr" %}
6727
6728 ins_encode(aarch64_enc_ldr(dst, mem));
6729
6730 ins_pipe(iload_reg_mem);
6731 %}
6732
6733 // Load Compressed Pointer
6734 instruct loadN(iRegNNoSp dst, memory4 mem)
6735 %{
6736 match(Set dst (LoadN mem));
6737 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6738
6739 ins_cost(4 * INSN_COST);
6740 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6741
6742 ins_encode(aarch64_enc_ldrw(dst, mem));
6743
6744 ins_pipe(iload_reg_mem);
6745 %}
6746
6747 // Load Klass Pointer
6748 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6749 %{
6750 match(Set dst (LoadKlass mem));
6751 predicate(!needs_acquiring_load(n));
6752
6753 ins_cost(4 * INSN_COST);
6754 format %{ "ldr $dst, $mem\t# class" %}
6755
6756 ins_encode(aarch64_enc_ldr(dst, mem));
6757
6758 ins_pipe(iload_reg_mem);
6759 %}
6760
6761 // Load Narrow Klass Pointer
6762 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6763 %{
6764 match(Set dst (LoadNKlass mem));
6765 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6766
6767 ins_cost(4 * INSN_COST);
6768 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6769
6770 ins_encode(aarch64_enc_ldrw(dst, mem));
6771
6772 ins_pipe(iload_reg_mem);
6773 %}
6774
6775 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6776 %{
6777 match(Set dst (LoadNKlass mem));
6778 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6779
6780 ins_cost(4 * INSN_COST);
6781 format %{
6782 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6783 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6784 %}
6785 ins_encode %{
6786 // inlined aarch64_enc_ldrw
6787 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6788 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6789 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6790 %}
6791 ins_pipe(iload_reg_mem);
6792 %}
6793
6794 // Load Float
6795 instruct loadF(vRegF dst, memory4 mem)
6796 %{
6797 match(Set dst (LoadF mem));
6798 predicate(!needs_acquiring_load(n));
6799
6800 ins_cost(4 * INSN_COST);
6801 format %{ "ldrs $dst, $mem\t# float" %}
6802
6803 ins_encode( aarch64_enc_ldrs(dst, mem) );
6804
6805 ins_pipe(pipe_class_memory);
6806 %}
6807
6808 // Load Double
6809 instruct loadD(vRegD dst, memory8 mem)
6810 %{
6811 match(Set dst (LoadD mem));
6812 predicate(!needs_acquiring_load(n));
6813
6814 ins_cost(4 * INSN_COST);
6815 format %{ "ldrd $dst, $mem\t# double" %}
6816
6817 ins_encode( aarch64_enc_ldrd(dst, mem) );
6818
6819 ins_pipe(pipe_class_memory);
6820 %}
6821
6822
6823 // Load Int Constant
6824 instruct loadConI(iRegINoSp dst, immI src)
6825 %{
6826 match(Set dst src);
6827
6828 ins_cost(INSN_COST);
6829 format %{ "mov $dst, $src\t# int" %}
6830
6831 ins_encode( aarch64_enc_movw_imm(dst, src) );
6832
6833 ins_pipe(ialu_imm);
6834 %}
6835
6836 // Load Long Constant
6837 instruct loadConL(iRegLNoSp dst, immL src)
6838 %{
6839 match(Set dst src);
6840
6841 ins_cost(INSN_COST);
6842 format %{ "mov $dst, $src\t# long" %}
6843
6844 ins_encode( aarch64_enc_mov_imm(dst, src) );
6845
6846 ins_pipe(ialu_imm);
6847 %}
6848
6849 // Load Pointer Constant
6850
6851 instruct loadConP(iRegPNoSp dst, immP con)
6852 %{
6853 match(Set dst con);
6854
6855 ins_cost(INSN_COST * 4);
6856 format %{
6857 "mov $dst, $con\t# ptr\n\t"
6858 %}
6859
6860 ins_encode(aarch64_enc_mov_p(dst, con));
6861
6862 ins_pipe(ialu_imm);
6863 %}
6864
6865 // Load Null Pointer Constant
6866
6867 instruct loadConP0(iRegPNoSp dst, immP0 con)
6868 %{
6869 match(Set dst con);
6870
6871 ins_cost(INSN_COST);
6872 format %{ "mov $dst, $con\t# nullptr ptr" %}
6873
6874 ins_encode(aarch64_enc_mov_p0(dst, con));
6875
6876 ins_pipe(ialu_imm);
6877 %}
6878
6879 // Load Pointer Constant One
6880
6881 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6882 %{
6883 match(Set dst con);
6884
6885 ins_cost(INSN_COST);
6886 format %{ "mov $dst, $con\t# nullptr ptr" %}
6887
6888 ins_encode(aarch64_enc_mov_p1(dst, con));
6889
6890 ins_pipe(ialu_imm);
6891 %}
6892
6893 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6894 %{
6895 match(Set dst con);
6896
6897 ins_cost(INSN_COST);
6898 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6899
6900 ins_encode %{
6901 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6902 %}
6903
6904 ins_pipe(ialu_imm);
6905 %}
6906
6907 // Load Narrow Pointer Constant
6908
6909 instruct loadConN(iRegNNoSp dst, immN con)
6910 %{
6911 match(Set dst con);
6912
6913 ins_cost(INSN_COST * 4);
6914 format %{ "mov $dst, $con\t# compressed ptr" %}
6915
6916 ins_encode(aarch64_enc_mov_n(dst, con));
6917
6918 ins_pipe(ialu_imm);
6919 %}
6920
6921 // Load Narrow Null Pointer Constant
6922
6923 instruct loadConN0(iRegNNoSp dst, immN0 con)
6924 %{
6925 match(Set dst con);
6926
6927 ins_cost(INSN_COST);
6928 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6929
6930 ins_encode(aarch64_enc_mov_n0(dst, con));
6931
6932 ins_pipe(ialu_imm);
6933 %}
6934
6935 // Load Narrow Klass Constant
6936
6937 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6938 %{
6939 match(Set dst con);
6940
6941 ins_cost(INSN_COST);
6942 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6943
6944 ins_encode(aarch64_enc_mov_nk(dst, con));
6945
6946 ins_pipe(ialu_imm);
6947 %}
6948
6949 // Load Packed Float Constant
6950
6951 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6952 match(Set dst con);
6953 ins_cost(INSN_COST * 4);
6954 format %{ "fmovs $dst, $con"%}
6955 ins_encode %{
6956 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6957 %}
6958
6959 ins_pipe(fp_imm_s);
6960 %}
6961
6962 // Load Float Constant
6963
6964 instruct loadConF(vRegF dst, immF con) %{
6965 match(Set dst con);
6966
6967 ins_cost(INSN_COST * 4);
6968
6969 format %{
6970 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6971 %}
6972
6973 ins_encode %{
6974 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6975 %}
6976
6977 ins_pipe(fp_load_constant_s);
6978 %}
6979
6980 // Load Packed Double Constant
6981
6982 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6983 match(Set dst con);
6984 ins_cost(INSN_COST);
6985 format %{ "fmovd $dst, $con"%}
6986 ins_encode %{
6987 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6988 %}
6989
6990 ins_pipe(fp_imm_d);
6991 %}
6992
6993 // Load Double Constant
6994
6995 instruct loadConD(vRegD dst, immD con) %{
6996 match(Set dst con);
6997
6998 ins_cost(INSN_COST * 5);
6999 format %{
7000 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7001 %}
7002
7003 ins_encode %{
7004 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7005 %}
7006
7007 ins_pipe(fp_load_constant_d);
7008 %}
7009
7010 // Load Half Float Constant
7011 instruct loadConH(vRegF dst, immH con) %{
7012 match(Set dst con);
7013 format %{ "mov rscratch1, $con\n\t"
7014 "fmov $dst, rscratch1"
7015 %}
7016 ins_encode %{
7017 __ movw(rscratch1, (uint32_t)$con$$constant);
7018 __ fmovs($dst$$FloatRegister, rscratch1);
7019 %}
7020 ins_pipe(pipe_class_default);
7021 %}
7022
7023 // Store Instructions
7024
7025 // Store Byte
7026 instruct storeB(iRegIorL2I src, memory1 mem)
7027 %{
7028 match(Set mem (StoreB mem src));
7029 predicate(!needs_releasing_store(n));
7030
7031 ins_cost(INSN_COST);
7032 format %{ "strb $src, $mem\t# byte" %}
7033
7034 ins_encode(aarch64_enc_strb(src, mem));
7035
7036 ins_pipe(istore_reg_mem);
7037 %}
7038
7039
7040 instruct storeimmB0(immI0 zero, memory1 mem)
7041 %{
7042 match(Set mem (StoreB mem zero));
7043 predicate(!needs_releasing_store(n));
7044
7045 ins_cost(INSN_COST);
7046 format %{ "strb rscractch2, $mem\t# byte" %}
7047
7048 ins_encode(aarch64_enc_strb0(mem));
7049
7050 ins_pipe(istore_mem);
7051 %}
7052
7053 // Store Char/Short
7054 instruct storeC(iRegIorL2I src, memory2 mem)
7055 %{
7056 match(Set mem (StoreC mem src));
7057 predicate(!needs_releasing_store(n));
7058
7059 ins_cost(INSN_COST);
7060 format %{ "strh $src, $mem\t# short" %}
7061
7062 ins_encode(aarch64_enc_strh(src, mem));
7063
7064 ins_pipe(istore_reg_mem);
7065 %}
7066
7067 instruct storeimmC0(immI0 zero, memory2 mem)
7068 %{
7069 match(Set mem (StoreC mem zero));
7070 predicate(!needs_releasing_store(n));
7071
7072 ins_cost(INSN_COST);
7073 format %{ "strh zr, $mem\t# short" %}
7074
7075 ins_encode(aarch64_enc_strh0(mem));
7076
7077 ins_pipe(istore_mem);
7078 %}
7079
7080 // Store Integer
7081
7082 instruct storeI(iRegIorL2I src, memory4 mem)
7083 %{
7084 match(Set mem(StoreI mem src));
7085 predicate(!needs_releasing_store(n));
7086
7087 ins_cost(INSN_COST);
7088 format %{ "strw $src, $mem\t# int" %}
7089
7090 ins_encode(aarch64_enc_strw(src, mem));
7091
7092 ins_pipe(istore_reg_mem);
7093 %}
7094
7095 instruct storeimmI0(immI0 zero, memory4 mem)
7096 %{
7097 match(Set mem(StoreI mem zero));
7098 predicate(!needs_releasing_store(n));
7099
7100 ins_cost(INSN_COST);
7101 format %{ "strw zr, $mem\t# int" %}
7102
7103 ins_encode(aarch64_enc_strw0(mem));
7104
7105 ins_pipe(istore_mem);
7106 %}
7107
7108 // Store Long (64 bit signed)
7109 instruct storeL(iRegL src, memory8 mem)
7110 %{
7111 match(Set mem (StoreL mem src));
7112 predicate(!needs_releasing_store(n));
7113
7114 ins_cost(INSN_COST);
7115 format %{ "str $src, $mem\t# int" %}
7116
7117 ins_encode(aarch64_enc_str(src, mem));
7118
7119 ins_pipe(istore_reg_mem);
7120 %}
7121
7122 // Store Long (64 bit signed)
7123 instruct storeimmL0(immL0 zero, memory8 mem)
7124 %{
7125 match(Set mem (StoreL mem zero));
7126 predicate(!needs_releasing_store(n));
7127
7128 ins_cost(INSN_COST);
7129 format %{ "str zr, $mem\t# int" %}
7130
7131 ins_encode(aarch64_enc_str0(mem));
7132
7133 ins_pipe(istore_mem);
7134 %}
7135
7136 // Store Pointer
7137 instruct storeP(iRegP src, memory8 mem)
7138 %{
7139 match(Set mem (StoreP mem src));
7140 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7141
7142 ins_cost(INSN_COST);
7143 format %{ "str $src, $mem\t# ptr" %}
7144
7145 ins_encode(aarch64_enc_str(src, mem));
7146
7147 ins_pipe(istore_reg_mem);
7148 %}
7149
7150 // Store Pointer
7151 instruct storeimmP0(immP0 zero, memory8 mem)
7152 %{
7153 match(Set mem (StoreP mem zero));
7154 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7155
7156 ins_cost(INSN_COST);
7157 format %{ "str zr, $mem\t# ptr" %}
7158
7159 ins_encode(aarch64_enc_str0(mem));
7160
7161 ins_pipe(istore_mem);
7162 %}
7163
7164 // Store Compressed Pointer
7165 instruct storeN(iRegN src, memory4 mem)
7166 %{
7167 match(Set mem (StoreN mem src));
7168 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7169
7170 ins_cost(INSN_COST);
7171 format %{ "strw $src, $mem\t# compressed ptr" %}
7172
7173 ins_encode(aarch64_enc_strw(src, mem));
7174
7175 ins_pipe(istore_reg_mem);
7176 %}
7177
7178 instruct storeImmN0(immN0 zero, memory4 mem)
7179 %{
7180 match(Set mem (StoreN mem zero));
7181 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7182
7183 ins_cost(INSN_COST);
7184 format %{ "strw zr, $mem\t# compressed ptr" %}
7185
7186 ins_encode(aarch64_enc_strw0(mem));
7187
7188 ins_pipe(istore_mem);
7189 %}
7190
7191 // Store Float
7192 instruct storeF(vRegF src, memory4 mem)
7193 %{
7194 match(Set mem (StoreF mem src));
7195 predicate(!needs_releasing_store(n));
7196
7197 ins_cost(INSN_COST);
7198 format %{ "strs $src, $mem\t# float" %}
7199
7200 ins_encode( aarch64_enc_strs(src, mem) );
7201
7202 ins_pipe(pipe_class_memory);
7203 %}
7204
7205 // TODO
7206 // implement storeImmF0 and storeFImmPacked
7207
7208 // Store Double
7209 instruct storeD(vRegD src, memory8 mem)
7210 %{
7211 match(Set mem (StoreD mem src));
7212 predicate(!needs_releasing_store(n));
7213
7214 ins_cost(INSN_COST);
7215 format %{ "strd $src, $mem\t# double" %}
7216
7217 ins_encode( aarch64_enc_strd(src, mem) );
7218
7219 ins_pipe(pipe_class_memory);
7220 %}
7221
7222 // Store Compressed Klass Pointer
7223 instruct storeNKlass(iRegN src, memory4 mem)
7224 %{
7225 predicate(!needs_releasing_store(n));
7226 match(Set mem (StoreNKlass mem src));
7227
7228 ins_cost(INSN_COST);
7229 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7230
7231 ins_encode(aarch64_enc_strw(src, mem));
7232
7233 ins_pipe(istore_reg_mem);
7234 %}
7235
7236 // TODO
7237 // implement storeImmD0 and storeDImmPacked
7238
7239 // prefetch instructions
7240 // Must be safe to execute with invalid address (cannot fault).
7241
7242 instruct prefetchalloc( memory8 mem ) %{
7243 match(PrefetchAllocation mem);
7244
7245 ins_cost(INSN_COST);
7246 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7247
7248 ins_encode( aarch64_enc_prefetchw(mem) );
7249
7250 ins_pipe(iload_prefetch);
7251 %}
7252
7253 // ---------------- volatile loads and stores ----------------
7254
7255 // Load Byte (8 bit signed)
7256 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7257 %{
7258 match(Set dst (LoadB mem));
7259
7260 ins_cost(VOLATILE_REF_COST);
7261 format %{ "ldarsb $dst, $mem\t# byte" %}
7262
7263 ins_encode(aarch64_enc_ldarsb(dst, mem));
7264
7265 ins_pipe(pipe_serial);
7266 %}
7267
7268 // Load Byte (8 bit signed) into long
7269 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7270 %{
7271 match(Set dst (ConvI2L (LoadB mem)));
7272
7273 ins_cost(VOLATILE_REF_COST);
7274 format %{ "ldarsb $dst, $mem\t# byte" %}
7275
7276 ins_encode(aarch64_enc_ldarsb(dst, mem));
7277
7278 ins_pipe(pipe_serial);
7279 %}
7280
7281 // Load Byte (8 bit unsigned)
7282 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7283 %{
7284 match(Set dst (LoadUB mem));
7285
7286 ins_cost(VOLATILE_REF_COST);
7287 format %{ "ldarb $dst, $mem\t# byte" %}
7288
7289 ins_encode(aarch64_enc_ldarb(dst, mem));
7290
7291 ins_pipe(pipe_serial);
7292 %}
7293
7294 // Load Byte (8 bit unsigned) into long
7295 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7296 %{
7297 match(Set dst (ConvI2L (LoadUB mem)));
7298
7299 ins_cost(VOLATILE_REF_COST);
7300 format %{ "ldarb $dst, $mem\t# byte" %}
7301
7302 ins_encode(aarch64_enc_ldarb(dst, mem));
7303
7304 ins_pipe(pipe_serial);
7305 %}
7306
7307 // Load Short (16 bit signed)
7308 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7309 %{
7310 match(Set dst (LoadS mem));
7311
7312 ins_cost(VOLATILE_REF_COST);
7313 format %{ "ldarshw $dst, $mem\t# short" %}
7314
7315 ins_encode(aarch64_enc_ldarshw(dst, mem));
7316
7317 ins_pipe(pipe_serial);
7318 %}
7319
7320 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7321 %{
7322 match(Set dst (LoadUS mem));
7323
7324 ins_cost(VOLATILE_REF_COST);
7325 format %{ "ldarhw $dst, $mem\t# short" %}
7326
7327 ins_encode(aarch64_enc_ldarhw(dst, mem));
7328
7329 ins_pipe(pipe_serial);
7330 %}
7331
7332 // Load Short/Char (16 bit unsigned) into long
7333 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7334 %{
7335 match(Set dst (ConvI2L (LoadUS mem)));
7336
7337 ins_cost(VOLATILE_REF_COST);
7338 format %{ "ldarh $dst, $mem\t# short" %}
7339
7340 ins_encode(aarch64_enc_ldarh(dst, mem));
7341
7342 ins_pipe(pipe_serial);
7343 %}
7344
7345 // Load Short/Char (16 bit signed) into long
7346 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7347 %{
7348 match(Set dst (ConvI2L (LoadS mem)));
7349
7350 ins_cost(VOLATILE_REF_COST);
7351 format %{ "ldarh $dst, $mem\t# short" %}
7352
7353 ins_encode(aarch64_enc_ldarsh(dst, mem));
7354
7355 ins_pipe(pipe_serial);
7356 %}
7357
7358 // Load Integer (32 bit signed)
7359 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7360 %{
7361 match(Set dst (LoadI mem));
7362
7363 ins_cost(VOLATILE_REF_COST);
7364 format %{ "ldarw $dst, $mem\t# int" %}
7365
7366 ins_encode(aarch64_enc_ldarw(dst, mem));
7367
7368 ins_pipe(pipe_serial);
7369 %}
7370
7371 // Load Integer (32 bit unsigned) into long
7372 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7373 %{
7374 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7375
7376 ins_cost(VOLATILE_REF_COST);
7377 format %{ "ldarw $dst, $mem\t# int" %}
7378
7379 ins_encode(aarch64_enc_ldarw(dst, mem));
7380
7381 ins_pipe(pipe_serial);
7382 %}
7383
7384 // Load Long (64 bit signed)
7385 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7386 %{
7387 match(Set dst (LoadL mem));
7388
7389 ins_cost(VOLATILE_REF_COST);
7390 format %{ "ldar $dst, $mem\t# int" %}
7391
7392 ins_encode(aarch64_enc_ldar(dst, mem));
7393
7394 ins_pipe(pipe_serial);
7395 %}
7396
7397 // Load Pointer
7398 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7399 %{
7400 match(Set dst (LoadP mem));
7401 predicate(n->as_Load()->barrier_data() == 0);
7402
7403 ins_cost(VOLATILE_REF_COST);
7404 format %{ "ldar $dst, $mem\t# ptr" %}
7405
7406 ins_encode(aarch64_enc_ldar(dst, mem));
7407
7408 ins_pipe(pipe_serial);
7409 %}
7410
7411 // Load Compressed Pointer
7412 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7413 %{
7414 match(Set dst (LoadN mem));
7415 predicate(n->as_Load()->barrier_data() == 0);
7416
7417 ins_cost(VOLATILE_REF_COST);
7418 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7419
7420 ins_encode(aarch64_enc_ldarw(dst, mem));
7421
7422 ins_pipe(pipe_serial);
7423 %}
7424
7425 // Load Float
7426 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7427 %{
7428 match(Set dst (LoadF mem));
7429
7430 ins_cost(VOLATILE_REF_COST);
7431 format %{ "ldars $dst, $mem\t# float" %}
7432
7433 ins_encode( aarch64_enc_fldars(dst, mem) );
7434
7435 ins_pipe(pipe_serial);
7436 %}
7437
7438 // Load Double
7439 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7440 %{
7441 match(Set dst (LoadD mem));
7442
7443 ins_cost(VOLATILE_REF_COST);
7444 format %{ "ldard $dst, $mem\t# double" %}
7445
7446 ins_encode( aarch64_enc_fldard(dst, mem) );
7447
7448 ins_pipe(pipe_serial);
7449 %}
7450
7451 // Store Byte
7452 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7453 %{
7454 match(Set mem (StoreB mem src));
7455
7456 ins_cost(VOLATILE_REF_COST);
7457 format %{ "stlrb $src, $mem\t# byte" %}
7458
7459 ins_encode(aarch64_enc_stlrb(src, mem));
7460
7461 ins_pipe(pipe_class_memory);
7462 %}
7463
7464 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7465 %{
7466 match(Set mem (StoreB mem zero));
7467
7468 ins_cost(VOLATILE_REF_COST);
7469 format %{ "stlrb zr, $mem\t# byte" %}
7470
7471 ins_encode(aarch64_enc_stlrb0(mem));
7472
7473 ins_pipe(pipe_class_memory);
7474 %}
7475
7476 // Store Char/Short
7477 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7478 %{
7479 match(Set mem (StoreC mem src));
7480
7481 ins_cost(VOLATILE_REF_COST);
7482 format %{ "stlrh $src, $mem\t# short" %}
7483
7484 ins_encode(aarch64_enc_stlrh(src, mem));
7485
7486 ins_pipe(pipe_class_memory);
7487 %}
7488
7489 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7490 %{
7491 match(Set mem (StoreC mem zero));
7492
7493 ins_cost(VOLATILE_REF_COST);
7494 format %{ "stlrh zr, $mem\t# short" %}
7495
7496 ins_encode(aarch64_enc_stlrh0(mem));
7497
7498 ins_pipe(pipe_class_memory);
7499 %}
7500
7501 // Store Integer
7502
7503 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7504 %{
7505 match(Set mem(StoreI mem src));
7506
7507 ins_cost(VOLATILE_REF_COST);
7508 format %{ "stlrw $src, $mem\t# int" %}
7509
7510 ins_encode(aarch64_enc_stlrw(src, mem));
7511
7512 ins_pipe(pipe_class_memory);
7513 %}
7514
7515 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7516 %{
7517 match(Set mem(StoreI mem zero));
7518
7519 ins_cost(VOLATILE_REF_COST);
7520 format %{ "stlrw zr, $mem\t# int" %}
7521
7522 ins_encode(aarch64_enc_stlrw0(mem));
7523
7524 ins_pipe(pipe_class_memory);
7525 %}
7526
7527 // Store Long (64 bit signed)
7528 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7529 %{
7530 match(Set mem (StoreL mem src));
7531
7532 ins_cost(VOLATILE_REF_COST);
7533 format %{ "stlr $src, $mem\t# int" %}
7534
7535 ins_encode(aarch64_enc_stlr(src, mem));
7536
7537 ins_pipe(pipe_class_memory);
7538 %}
7539
7540 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7541 %{
7542 match(Set mem (StoreL mem zero));
7543
7544 ins_cost(VOLATILE_REF_COST);
7545 format %{ "stlr zr, $mem\t# int" %}
7546
7547 ins_encode(aarch64_enc_stlr0(mem));
7548
7549 ins_pipe(pipe_class_memory);
7550 %}
7551
7552 // Store Pointer
7553 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7554 %{
7555 match(Set mem (StoreP mem src));
7556 predicate(n->as_Store()->barrier_data() == 0);
7557
7558 ins_cost(VOLATILE_REF_COST);
7559 format %{ "stlr $src, $mem\t# ptr" %}
7560
7561 ins_encode(aarch64_enc_stlr(src, mem));
7562
7563 ins_pipe(pipe_class_memory);
7564 %}
7565
7566 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7567 %{
7568 match(Set mem (StoreP mem zero));
7569 predicate(n->as_Store()->barrier_data() == 0);
7570
7571 ins_cost(VOLATILE_REF_COST);
7572 format %{ "stlr zr, $mem\t# ptr" %}
7573
7574 ins_encode(aarch64_enc_stlr0(mem));
7575
7576 ins_pipe(pipe_class_memory);
7577 %}
7578
7579 // Store Compressed Pointer
7580 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7581 %{
7582 match(Set mem (StoreN mem src));
7583 predicate(n->as_Store()->barrier_data() == 0);
7584
7585 ins_cost(VOLATILE_REF_COST);
7586 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7587
7588 ins_encode(aarch64_enc_stlrw(src, mem));
7589
7590 ins_pipe(pipe_class_memory);
7591 %}
7592
7593 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7594 %{
7595 match(Set mem (StoreN mem zero));
7596 predicate(n->as_Store()->barrier_data() == 0);
7597
7598 ins_cost(VOLATILE_REF_COST);
7599 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7600
7601 ins_encode(aarch64_enc_stlrw0(mem));
7602
7603 ins_pipe(pipe_class_memory);
7604 %}
7605
7606 // Store Float
7607 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7608 %{
7609 match(Set mem (StoreF mem src));
7610
7611 ins_cost(VOLATILE_REF_COST);
7612 format %{ "stlrs $src, $mem\t# float" %}
7613
7614 ins_encode( aarch64_enc_fstlrs(src, mem) );
7615
7616 ins_pipe(pipe_class_memory);
7617 %}
7618
7619 // TODO
7620 // implement storeImmF0 and storeFImmPacked
7621
7622 // Store Double
7623 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7624 %{
7625 match(Set mem (StoreD mem src));
7626
7627 ins_cost(VOLATILE_REF_COST);
7628 format %{ "stlrd $src, $mem\t# double" %}
7629
7630 ins_encode( aarch64_enc_fstlrd(src, mem) );
7631
7632 ins_pipe(pipe_class_memory);
7633 %}
7634
7635 // ---------------- end of volatile loads and stores ----------------
7636
7637 instruct cacheWB(indirect addr)
7638 %{
7639 predicate(VM_Version::supports_data_cache_line_flush());
7640 match(CacheWB addr);
7641
7642 ins_cost(100);
7643 format %{"cache wb $addr" %}
7644 ins_encode %{
7645 assert($addr->index_position() < 0, "should be");
7646 assert($addr$$disp == 0, "should be");
7647 __ cache_wb(Address($addr$$base$$Register, 0));
7648 %}
7649 ins_pipe(pipe_slow); // XXX
7650 %}
7651
7652 instruct cacheWBPreSync()
7653 %{
7654 predicate(VM_Version::supports_data_cache_line_flush());
7655 match(CacheWBPreSync);
7656
7657 ins_cost(100);
7658 format %{"cache wb presync" %}
7659 ins_encode %{
7660 __ cache_wbsync(true);
7661 %}
7662 ins_pipe(pipe_slow); // XXX
7663 %}
7664
7665 instruct cacheWBPostSync()
7666 %{
7667 predicate(VM_Version::supports_data_cache_line_flush());
7668 match(CacheWBPostSync);
7669
7670 ins_cost(100);
7671 format %{"cache wb postsync" %}
7672 ins_encode %{
7673 __ cache_wbsync(false);
7674 %}
7675 ins_pipe(pipe_slow); // XXX
7676 %}
7677
7678 // ============================================================================
7679 // BSWAP Instructions
7680
7681 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7682 match(Set dst (ReverseBytesI src));
7683
7684 ins_cost(INSN_COST);
7685 format %{ "revw $dst, $src" %}
7686
7687 ins_encode %{
7688 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7689 %}
7690
7691 ins_pipe(ialu_reg);
7692 %}
7693
7694 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7695 match(Set dst (ReverseBytesL src));
7696
7697 ins_cost(INSN_COST);
7698 format %{ "rev $dst, $src" %}
7699
7700 ins_encode %{
7701 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7702 %}
7703
7704 ins_pipe(ialu_reg);
7705 %}
7706
7707 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7708 match(Set dst (ReverseBytesUS src));
7709
7710 ins_cost(INSN_COST);
7711 format %{ "rev16w $dst, $src\t# $dst -> unsigned short" %}
7712
7713 ins_encode %{
7714 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7715 __ narrow_subword_type(as_Register($dst$$reg), T_CHAR);
7716 %}
7717
7718 ins_pipe(ialu_reg);
7719 %}
7720
7721 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7722 match(Set dst (ReverseBytesS src));
7723
7724 ins_cost(INSN_COST);
7725 format %{ "rev16w $dst, $src\n\t"
7726 "sbfmw $dst, $dst, #0, #15" %}
7727
7728 ins_encode %{
7729 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7730 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7731 %}
7732
7733 ins_pipe(ialu_reg);
7734 %}
7735
7736 // ============================================================================
7737 // Zero Count Instructions
7738
7739 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7740 match(Set dst (CountLeadingZerosI src));
7741
7742 ins_cost(INSN_COST);
7743 format %{ "clzw $dst, $src" %}
7744 ins_encode %{
7745 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7746 %}
7747
7748 ins_pipe(ialu_reg);
7749 %}
7750
7751 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7752 match(Set dst (CountLeadingZerosL src));
7753
7754 ins_cost(INSN_COST);
7755 format %{ "clz $dst, $src" %}
7756 ins_encode %{
7757 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7758 %}
7759
7760 ins_pipe(ialu_reg);
7761 %}
7762
7763 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7764 match(Set dst (CountTrailingZerosI src));
7765
7766 ins_cost(INSN_COST * 2);
7767 format %{ "rbitw $dst, $src\n\t"
7768 "clzw $dst, $dst" %}
7769 ins_encode %{
7770 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7771 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7772 %}
7773
7774 ins_pipe(ialu_reg);
7775 %}
7776
7777 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7778 match(Set dst (CountTrailingZerosL src));
7779
7780 ins_cost(INSN_COST * 2);
7781 format %{ "rbit $dst, $src\n\t"
7782 "clz $dst, $dst" %}
7783 ins_encode %{
7784 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7785 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7786 %}
7787
7788 ins_pipe(ialu_reg);
7789 %}
7790
7791 //---------- Population Count Instructions -------------------------------------
7792 //
7793
7794 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7795 match(Set dst (PopCountI src));
7796 effect(TEMP tmp);
7797 ins_cost(INSN_COST * 13);
7798
7799 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7800 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7801 "addv $tmp, $tmp\t# vector (8B)\n\t"
7802 "mov $dst, $tmp\t# vector (1D)" %}
7803 ins_encode %{
7804 __ fmovs($tmp$$FloatRegister, $src$$Register);
7805 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7806 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7807 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7808 %}
7809
7810 ins_pipe(pipe_class_default);
7811 %}
7812
7813 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7814 match(Set dst (PopCountI (LoadI mem)));
7815 effect(TEMP tmp);
7816 ins_cost(INSN_COST * 13);
7817
7818 format %{ "ldrs $tmp, $mem\n\t"
7819 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7820 "addv $tmp, $tmp\t# vector (8B)\n\t"
7821 "mov $dst, $tmp\t# vector (1D)" %}
7822 ins_encode %{
7823 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7824 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7825 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7826 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7827 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7828 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7829 %}
7830
7831 ins_pipe(pipe_class_default);
7832 %}
7833
7834 // Note: Long.bitCount(long) returns an int.
7835 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7836 match(Set dst (PopCountL src));
7837 effect(TEMP tmp);
7838 ins_cost(INSN_COST * 13);
7839
7840 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7841 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7842 "addv $tmp, $tmp\t# vector (8B)\n\t"
7843 "mov $dst, $tmp\t# vector (1D)" %}
7844 ins_encode %{
7845 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7846 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7847 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7848 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7849 %}
7850
7851 ins_pipe(pipe_class_default);
7852 %}
7853
7854 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7855 match(Set dst (PopCountL (LoadL mem)));
7856 effect(TEMP tmp);
7857 ins_cost(INSN_COST * 13);
7858
7859 format %{ "ldrd $tmp, $mem\n\t"
7860 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7861 "addv $tmp, $tmp\t# vector (8B)\n\t"
7862 "mov $dst, $tmp\t# vector (1D)" %}
7863 ins_encode %{
7864 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7865 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7866 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7867 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7868 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7869 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7870 %}
7871
7872 ins_pipe(pipe_class_default);
7873 %}
7874
7875 // ============================================================================
7876 // VerifyVectorAlignment Instruction
7877
7878 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7879 match(Set addr (VerifyVectorAlignment addr mask));
7880 effect(KILL cr);
7881 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7882 ins_encode %{
7883 Label Lskip;
7884 // check if masked bits of addr are zero
7885 __ tst($addr$$Register, $mask$$constant);
7886 __ br(Assembler::EQ, Lskip);
7887 __ stop("verify_vector_alignment found a misaligned vector memory access");
7888 __ bind(Lskip);
7889 %}
7890 ins_pipe(pipe_slow);
7891 %}
7892
7893 // ============================================================================
7894 // MemBar Instruction
7895
7896 instruct load_fence() %{
7897 match(LoadFence);
7898 ins_cost(VOLATILE_REF_COST);
7899
7900 format %{ "load_fence" %}
7901
7902 ins_encode %{
7903 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7904 %}
7905 ins_pipe(pipe_serial);
7906 %}
7907
7908 instruct unnecessary_membar_acquire() %{
7909 predicate(unnecessary_acquire(n));
7910 match(MemBarAcquire);
7911 ins_cost(0);
7912
7913 format %{ "membar_acquire (elided)" %}
7914
7915 ins_encode %{
7916 __ block_comment("membar_acquire (elided)");
7917 %}
7918
7919 ins_pipe(pipe_class_empty);
7920 %}
7921
7922 instruct membar_acquire() %{
7923 match(MemBarAcquire);
7924 ins_cost(VOLATILE_REF_COST);
7925
7926 format %{ "membar_acquire\n\t"
7927 "dmb ishld" %}
7928
7929 ins_encode %{
7930 __ block_comment("membar_acquire");
7931 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7932 %}
7933
7934 ins_pipe(pipe_serial);
7935 %}
7936
7937
7938 instruct membar_acquire_lock() %{
7939 match(MemBarAcquireLock);
7940 ins_cost(VOLATILE_REF_COST);
7941
7942 format %{ "membar_acquire_lock (elided)" %}
7943
7944 ins_encode %{
7945 __ block_comment("membar_acquire_lock (elided)");
7946 %}
7947
7948 ins_pipe(pipe_serial);
7949 %}
7950
7951 instruct store_fence() %{
7952 match(StoreFence);
7953 ins_cost(VOLATILE_REF_COST);
7954
7955 format %{ "store_fence" %}
7956
7957 ins_encode %{
7958 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7959 %}
7960 ins_pipe(pipe_serial);
7961 %}
7962
7963 instruct unnecessary_membar_release() %{
7964 predicate(unnecessary_release(n));
7965 match(MemBarRelease);
7966 ins_cost(0);
7967
7968 format %{ "membar_release (elided)" %}
7969
7970 ins_encode %{
7971 __ block_comment("membar_release (elided)");
7972 %}
7973 ins_pipe(pipe_serial);
7974 %}
7975
7976 instruct membar_release() %{
7977 match(MemBarRelease);
7978 ins_cost(VOLATILE_REF_COST);
7979
7980 format %{ "membar_release\n\t"
7981 "dmb ishst\n\tdmb ishld" %}
7982
7983 ins_encode %{
7984 __ block_comment("membar_release");
7985 // These will be merged if AlwaysMergeDMB is enabled.
7986 __ membar(Assembler::StoreStore);
7987 __ membar(Assembler::LoadStore);
7988 %}
7989 ins_pipe(pipe_serial);
7990 %}
7991
7992 instruct membar_storestore() %{
7993 match(MemBarStoreStore);
7994 match(StoreStoreFence);
7995 ins_cost(VOLATILE_REF_COST);
7996
7997 format %{ "MEMBAR-store-store" %}
7998
7999 ins_encode %{
8000 __ membar(Assembler::StoreStore);
8001 %}
8002 ins_pipe(pipe_serial);
8003 %}
8004
8005 instruct membar_release_lock() %{
8006 match(MemBarReleaseLock);
8007 ins_cost(VOLATILE_REF_COST);
8008
8009 format %{ "membar_release_lock (elided)" %}
8010
8011 ins_encode %{
8012 __ block_comment("membar_release_lock (elided)");
8013 %}
8014
8015 ins_pipe(pipe_serial);
8016 %}
8017
8018 instruct membar_storeload() %{
8019 match(MemBarStoreLoad);
8020 ins_cost(VOLATILE_REF_COST*100);
8021
8022 format %{ "MEMBAR-store-load\n\t"
8023 "dmb ish" %}
8024
8025 ins_encode %{
8026 __ block_comment("membar_storeload");
8027 __ membar(Assembler::StoreLoad);
8028 %}
8029
8030 ins_pipe(pipe_serial);
8031 %}
8032
8033 instruct unnecessary_membar_volatile() %{
8034 predicate(unnecessary_volatile(n));
8035 match(MemBarVolatile);
8036 ins_cost(0);
8037
8038 format %{ "membar_volatile (elided)" %}
8039
8040 ins_encode %{
8041 __ block_comment("membar_volatile (elided)");
8042 %}
8043
8044 ins_pipe(pipe_serial);
8045 %}
8046
8047 instruct membar_volatile() %{
8048 match(MemBarVolatile);
8049 ins_cost(VOLATILE_REF_COST*100);
8050
8051 format %{ "membar_volatile\n\t"
8052 "dmb ish"%}
8053
8054 ins_encode %{
8055 __ block_comment("membar_volatile");
8056 __ membar(Assembler::StoreLoad);
8057 %}
8058
8059 ins_pipe(pipe_serial);
8060 %}
8061
8062 instruct membar_full() %{
8063 match(MemBarFull);
8064 ins_cost(VOLATILE_REF_COST*100);
8065
8066 format %{ "membar_full\n\t"
8067 "dmb ish" %}
8068 ins_encode %{
8069 __ block_comment("membar_full");
8070 __ membar(Assembler::AnyAny);
8071 %}
8072
8073 ins_pipe(pipe_serial);
8074 %}
8075
8076 // ============================================================================
8077 // Cast/Convert Instructions
8078
8079 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8080 match(Set dst (CastX2P src));
8081
8082 ins_cost(INSN_COST);
8083 format %{ "mov $dst, $src\t# long -> ptr" %}
8084
8085 ins_encode %{
8086 if ($dst$$reg != $src$$reg) {
8087 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8088 }
8089 %}
8090
8091 ins_pipe(ialu_reg);
8092 %}
8093
8094 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8095 match(Set dst (CastP2X src));
8096
8097 ins_cost(INSN_COST);
8098 format %{ "mov $dst, $src\t# ptr -> long" %}
8099
8100 ins_encode %{
8101 if ($dst$$reg != $src$$reg) {
8102 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8103 }
8104 %}
8105
8106 ins_pipe(ialu_reg);
8107 %}
8108
8109 // Convert oop into int for vectors alignment masking
8110 instruct convP2I(iRegINoSp dst, iRegP src) %{
8111 match(Set dst (ConvL2I (CastP2X src)));
8112
8113 ins_cost(INSN_COST);
8114 format %{ "movw $dst, $src\t# ptr -> int" %}
8115 ins_encode %{
8116 __ movw($dst$$Register, $src$$Register);
8117 %}
8118
8119 ins_pipe(ialu_reg);
8120 %}
8121
8122 // Convert compressed oop into int for vectors alignment masking
8123 // in case of 32bit oops (heap < 4Gb).
8124 instruct convN2I(iRegINoSp dst, iRegN src)
8125 %{
8126 predicate(CompressedOops::shift() == 0);
8127 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8128
8129 ins_cost(INSN_COST);
8130 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8131 ins_encode %{
8132 __ movw($dst$$Register, $src$$Register);
8133 %}
8134
8135 ins_pipe(ialu_reg);
8136 %}
8137
8138
8139 // Convert oop pointer into compressed form
8140 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8141 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8142 match(Set dst (EncodeP src));
8143 effect(KILL cr);
8144 ins_cost(INSN_COST * 3);
8145 format %{ "encode_heap_oop $dst, $src" %}
8146 ins_encode %{
8147 Register s = $src$$Register;
8148 Register d = $dst$$Register;
8149 __ encode_heap_oop(d, s);
8150 %}
8151 ins_pipe(ialu_reg);
8152 %}
8153
8154 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8155 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8156 match(Set dst (EncodeP src));
8157 ins_cost(INSN_COST * 3);
8158 format %{ "encode_heap_oop_not_null $dst, $src" %}
8159 ins_encode %{
8160 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8161 %}
8162 ins_pipe(ialu_reg);
8163 %}
8164
8165 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8166 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8167 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8168 match(Set dst (DecodeN src));
8169 ins_cost(INSN_COST * 3);
8170 format %{ "decode_heap_oop $dst, $src" %}
8171 ins_encode %{
8172 Register s = $src$$Register;
8173 Register d = $dst$$Register;
8174 __ decode_heap_oop(d, s);
8175 %}
8176 ins_pipe(ialu_reg);
8177 %}
8178
8179 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8180 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8181 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8182 match(Set dst (DecodeN src));
8183 ins_cost(INSN_COST * 3);
8184 format %{ "decode_heap_oop_not_null $dst, $src" %}
8185 ins_encode %{
8186 Register s = $src$$Register;
8187 Register d = $dst$$Register;
8188 __ decode_heap_oop_not_null(d, s);
8189 %}
8190 ins_pipe(ialu_reg);
8191 %}
8192
8193 // n.b. AArch64 implementations of encode_klass_not_null and
8194 // decode_klass_not_null do not modify the flags register so, unlike
8195 // Intel, we don't kill CR as a side effect here
8196
8197 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8198 match(Set dst (EncodePKlass src));
8199
8200 ins_cost(INSN_COST * 3);
8201 format %{ "encode_klass_not_null $dst,$src" %}
8202
8203 ins_encode %{
8204 Register src_reg = as_Register($src$$reg);
8205 Register dst_reg = as_Register($dst$$reg);
8206 __ encode_klass_not_null(dst_reg, src_reg);
8207 %}
8208
8209 ins_pipe(ialu_reg);
8210 %}
8211
8212 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8213 match(Set dst (DecodeNKlass src));
8214
8215 ins_cost(INSN_COST * 3);
8216 format %{ "decode_klass_not_null $dst,$src" %}
8217
8218 ins_encode %{
8219 Register src_reg = as_Register($src$$reg);
8220 Register dst_reg = as_Register($dst$$reg);
8221 if (dst_reg != src_reg) {
8222 __ decode_klass_not_null(dst_reg, src_reg);
8223 } else {
8224 __ decode_klass_not_null(dst_reg);
8225 }
8226 %}
8227
8228 ins_pipe(ialu_reg);
8229 %}
8230
8231 instruct checkCastPP(iRegPNoSp dst)
8232 %{
8233 match(Set dst (CheckCastPP dst));
8234
8235 size(0);
8236 format %{ "# checkcastPP of $dst" %}
8237 ins_encode(/* empty encoding */);
8238 ins_pipe(pipe_class_empty);
8239 %}
8240
8241 instruct castPP(iRegPNoSp dst)
8242 %{
8243 match(Set dst (CastPP dst));
8244
8245 size(0);
8246 format %{ "# castPP of $dst" %}
8247 ins_encode(/* empty encoding */);
8248 ins_pipe(pipe_class_empty);
8249 %}
8250
8251 instruct castII(iRegI dst)
8252 %{
8253 predicate(VerifyConstraintCasts == 0);
8254 match(Set dst (CastII dst));
8255
8256 size(0);
8257 format %{ "# castII of $dst" %}
8258 ins_encode(/* empty encoding */);
8259 ins_cost(0);
8260 ins_pipe(pipe_class_empty);
8261 %}
8262
8263 instruct castII_checked(iRegI dst, rFlagsReg cr)
8264 %{
8265 predicate(VerifyConstraintCasts > 0);
8266 match(Set dst (CastII dst));
8267 effect(KILL cr);
8268
8269 format %{ "# castII_checked of $dst" %}
8270 ins_encode %{
8271 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8272 %}
8273 ins_pipe(pipe_slow);
8274 %}
8275
8276 instruct castLL(iRegL dst)
8277 %{
8278 predicate(VerifyConstraintCasts == 0);
8279 match(Set dst (CastLL dst));
8280
8281 size(0);
8282 format %{ "# castLL of $dst" %}
8283 ins_encode(/* empty encoding */);
8284 ins_cost(0);
8285 ins_pipe(pipe_class_empty);
8286 %}
8287
8288 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8289 %{
8290 predicate(VerifyConstraintCasts > 0);
8291 match(Set dst (CastLL dst));
8292 effect(KILL cr);
8293
8294 format %{ "# castLL_checked of $dst" %}
8295 ins_encode %{
8296 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8297 %}
8298 ins_pipe(pipe_slow);
8299 %}
8300
8301 instruct castHH(vRegF dst)
8302 %{
8303 match(Set dst (CastHH dst));
8304 size(0);
8305 format %{ "# castHH of $dst" %}
8306 ins_encode(/* empty encoding */);
8307 ins_cost(0);
8308 ins_pipe(pipe_class_empty);
8309 %}
8310
8311 instruct castFF(vRegF dst)
8312 %{
8313 match(Set dst (CastFF dst));
8314
8315 size(0);
8316 format %{ "# castFF of $dst" %}
8317 ins_encode(/* empty encoding */);
8318 ins_cost(0);
8319 ins_pipe(pipe_class_empty);
8320 %}
8321
8322 instruct castDD(vRegD dst)
8323 %{
8324 match(Set dst (CastDD dst));
8325
8326 size(0);
8327 format %{ "# castDD of $dst" %}
8328 ins_encode(/* empty encoding */);
8329 ins_cost(0);
8330 ins_pipe(pipe_class_empty);
8331 %}
8332
8333 instruct castVV(vReg dst)
8334 %{
8335 match(Set dst (CastVV dst));
8336
8337 size(0);
8338 format %{ "# castVV of $dst" %}
8339 ins_encode(/* empty encoding */);
8340 ins_cost(0);
8341 ins_pipe(pipe_class_empty);
8342 %}
8343
8344 instruct castVVMask(pRegGov dst)
8345 %{
8346 match(Set dst (CastVV dst));
8347
8348 size(0);
8349 format %{ "# castVV of $dst" %}
8350 ins_encode(/* empty encoding */);
8351 ins_cost(0);
8352 ins_pipe(pipe_class_empty);
8353 %}
8354
8355 // Manifest a CmpU result in an integer register.
8356 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8357 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8358 %{
8359 match(Set dst (CmpU3 src1 src2));
8360 effect(KILL flags);
8361
8362 ins_cost(INSN_COST * 3);
8363 format %{
8364 "cmpw $src1, $src2\n\t"
8365 "csetw $dst, ne\n\t"
8366 "cnegw $dst, lo\t# CmpU3(reg)"
8367 %}
8368 ins_encode %{
8369 __ cmpw($src1$$Register, $src2$$Register);
8370 __ csetw($dst$$Register, Assembler::NE);
8371 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8372 %}
8373
8374 ins_pipe(pipe_class_default);
8375 %}
8376
8377 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8378 %{
8379 match(Set dst (CmpU3 src1 src2));
8380 effect(KILL flags);
8381
8382 ins_cost(INSN_COST * 3);
8383 format %{
8384 "subsw zr, $src1, $src2\n\t"
8385 "csetw $dst, ne\n\t"
8386 "cnegw $dst, lo\t# CmpU3(imm)"
8387 %}
8388 ins_encode %{
8389 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8390 __ csetw($dst$$Register, Assembler::NE);
8391 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8392 %}
8393
8394 ins_pipe(pipe_class_default);
8395 %}
8396
8397 // Manifest a CmpUL result in an integer register.
8398 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8399 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8400 %{
8401 match(Set dst (CmpUL3 src1 src2));
8402 effect(KILL flags);
8403
8404 ins_cost(INSN_COST * 3);
8405 format %{
8406 "cmp $src1, $src2\n\t"
8407 "csetw $dst, ne\n\t"
8408 "cnegw $dst, lo\t# CmpUL3(reg)"
8409 %}
8410 ins_encode %{
8411 __ cmp($src1$$Register, $src2$$Register);
8412 __ csetw($dst$$Register, Assembler::NE);
8413 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8414 %}
8415
8416 ins_pipe(pipe_class_default);
8417 %}
8418
8419 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8420 %{
8421 match(Set dst (CmpUL3 src1 src2));
8422 effect(KILL flags);
8423
8424 ins_cost(INSN_COST * 3);
8425 format %{
8426 "subs zr, $src1, $src2\n\t"
8427 "csetw $dst, ne\n\t"
8428 "cnegw $dst, lo\t# CmpUL3(imm)"
8429 %}
8430 ins_encode %{
8431 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8432 __ csetw($dst$$Register, Assembler::NE);
8433 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8434 %}
8435
8436 ins_pipe(pipe_class_default);
8437 %}
8438
8439 // Manifest a CmpL result in an integer register.
8440 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8441 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8442 %{
8443 match(Set dst (CmpL3 src1 src2));
8444 effect(KILL flags);
8445
8446 ins_cost(INSN_COST * 3);
8447 format %{
8448 "cmp $src1, $src2\n\t"
8449 "csetw $dst, ne\n\t"
8450 "cnegw $dst, lt\t# CmpL3(reg)"
8451 %}
8452 ins_encode %{
8453 __ cmp($src1$$Register, $src2$$Register);
8454 __ csetw($dst$$Register, Assembler::NE);
8455 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8456 %}
8457
8458 ins_pipe(pipe_class_default);
8459 %}
8460
8461 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8462 %{
8463 match(Set dst (CmpL3 src1 src2));
8464 effect(KILL flags);
8465
8466 ins_cost(INSN_COST * 3);
8467 format %{
8468 "subs zr, $src1, $src2\n\t"
8469 "csetw $dst, ne\n\t"
8470 "cnegw $dst, lt\t# CmpL3(imm)"
8471 %}
8472 ins_encode %{
8473 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8474 __ csetw($dst$$Register, Assembler::NE);
8475 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8476 %}
8477
8478 ins_pipe(pipe_class_default);
8479 %}
8480
8481 // ============================================================================
8482 // Conditional Move Instructions
8483
8484 // n.b. we have identical rules for both a signed compare op (cmpOp)
8485 // and an unsigned compare op (cmpOpU). it would be nice if we could
8486 // define an op class which merged both inputs and use it to type the
8487 // argument to a single rule. unfortunatelyt his fails because the
8488 // opclass does not live up to the COND_INTER interface of its
8489 // component operands. When the generic code tries to negate the
8490 // operand it ends up running the generci Machoper::negate method
8491 // which throws a ShouldNotHappen. So, we have to provide two flavours
8492 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8493
8494 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8495 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8496
8497 ins_cost(INSN_COST * 2);
8498 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8499
8500 ins_encode %{
8501 __ cselw(as_Register($dst$$reg),
8502 as_Register($src2$$reg),
8503 as_Register($src1$$reg),
8504 (Assembler::Condition)$cmp$$cmpcode);
8505 %}
8506
8507 ins_pipe(icond_reg_reg);
8508 %}
8509
8510 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8511 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8512
8513 ins_cost(INSN_COST * 2);
8514 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8515
8516 ins_encode %{
8517 __ cselw(as_Register($dst$$reg),
8518 as_Register($src2$$reg),
8519 as_Register($src1$$reg),
8520 (Assembler::Condition)$cmp$$cmpcode);
8521 %}
8522
8523 ins_pipe(icond_reg_reg);
8524 %}
8525
8526 // special cases where one arg is zero
8527
8528 // n.b. this is selected in preference to the rule above because it
8529 // avoids loading constant 0 into a source register
8530
8531 // TODO
8532 // we ought only to be able to cull one of these variants as the ideal
8533 // transforms ought always to order the zero consistently (to left/right?)
8534
8535 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8536 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8537
8538 ins_cost(INSN_COST * 2);
8539 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8540
8541 ins_encode %{
8542 __ cselw(as_Register($dst$$reg),
8543 as_Register($src$$reg),
8544 zr,
8545 (Assembler::Condition)$cmp$$cmpcode);
8546 %}
8547
8548 ins_pipe(icond_reg);
8549 %}
8550
8551 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8552 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8553
8554 ins_cost(INSN_COST * 2);
8555 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8556
8557 ins_encode %{
8558 __ cselw(as_Register($dst$$reg),
8559 as_Register($src$$reg),
8560 zr,
8561 (Assembler::Condition)$cmp$$cmpcode);
8562 %}
8563
8564 ins_pipe(icond_reg);
8565 %}
8566
8567 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8568 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8569
8570 ins_cost(INSN_COST * 2);
8571 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8572
8573 ins_encode %{
8574 __ cselw(as_Register($dst$$reg),
8575 zr,
8576 as_Register($src$$reg),
8577 (Assembler::Condition)$cmp$$cmpcode);
8578 %}
8579
8580 ins_pipe(icond_reg);
8581 %}
8582
8583 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8584 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8585
8586 ins_cost(INSN_COST * 2);
8587 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8588
8589 ins_encode %{
8590 __ cselw(as_Register($dst$$reg),
8591 zr,
8592 as_Register($src$$reg),
8593 (Assembler::Condition)$cmp$$cmpcode);
8594 %}
8595
8596 ins_pipe(icond_reg);
8597 %}
8598
8599 // special case for creating a boolean 0 or 1
8600
8601 // n.b. this is selected in preference to the rule above because it
8602 // avoids loading constants 0 and 1 into a source register
8603
8604 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8605 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8606
8607 ins_cost(INSN_COST * 2);
8608 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8609
8610 ins_encode %{
8611 // equivalently
8612 // cset(as_Register($dst$$reg),
8613 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8614 __ csincw(as_Register($dst$$reg),
8615 zr,
8616 zr,
8617 (Assembler::Condition)$cmp$$cmpcode);
8618 %}
8619
8620 ins_pipe(icond_none);
8621 %}
8622
8623 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8624 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8625
8626 ins_cost(INSN_COST * 2);
8627 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8628
8629 ins_encode %{
8630 // equivalently
8631 // cset(as_Register($dst$$reg),
8632 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8633 __ csincw(as_Register($dst$$reg),
8634 zr,
8635 zr,
8636 (Assembler::Condition)$cmp$$cmpcode);
8637 %}
8638
8639 ins_pipe(icond_none);
8640 %}
8641
8642 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8643 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8644
8645 ins_cost(INSN_COST * 2);
8646 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8647
8648 ins_encode %{
8649 __ csel(as_Register($dst$$reg),
8650 as_Register($src2$$reg),
8651 as_Register($src1$$reg),
8652 (Assembler::Condition)$cmp$$cmpcode);
8653 %}
8654
8655 ins_pipe(icond_reg_reg);
8656 %}
8657
8658 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8659 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8660
8661 ins_cost(INSN_COST * 2);
8662 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8663
8664 ins_encode %{
8665 __ csel(as_Register($dst$$reg),
8666 as_Register($src2$$reg),
8667 as_Register($src1$$reg),
8668 (Assembler::Condition)$cmp$$cmpcode);
8669 %}
8670
8671 ins_pipe(icond_reg_reg);
8672 %}
8673
8674 // special cases where one arg is zero
8675
8676 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8677 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8678
8679 ins_cost(INSN_COST * 2);
8680 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8681
8682 ins_encode %{
8683 __ csel(as_Register($dst$$reg),
8684 zr,
8685 as_Register($src$$reg),
8686 (Assembler::Condition)$cmp$$cmpcode);
8687 %}
8688
8689 ins_pipe(icond_reg);
8690 %}
8691
8692 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8693 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8694
8695 ins_cost(INSN_COST * 2);
8696 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8697
8698 ins_encode %{
8699 __ csel(as_Register($dst$$reg),
8700 zr,
8701 as_Register($src$$reg),
8702 (Assembler::Condition)$cmp$$cmpcode);
8703 %}
8704
8705 ins_pipe(icond_reg);
8706 %}
8707
8708 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8709 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8710
8711 ins_cost(INSN_COST * 2);
8712 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8713
8714 ins_encode %{
8715 __ csel(as_Register($dst$$reg),
8716 as_Register($src$$reg),
8717 zr,
8718 (Assembler::Condition)$cmp$$cmpcode);
8719 %}
8720
8721 ins_pipe(icond_reg);
8722 %}
8723
8724 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8725 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8726
8727 ins_cost(INSN_COST * 2);
8728 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8729
8730 ins_encode %{
8731 __ csel(as_Register($dst$$reg),
8732 as_Register($src$$reg),
8733 zr,
8734 (Assembler::Condition)$cmp$$cmpcode);
8735 %}
8736
8737 ins_pipe(icond_reg);
8738 %}
8739
8740 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8741 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8742
8743 ins_cost(INSN_COST * 2);
8744 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8745
8746 ins_encode %{
8747 __ csel(as_Register($dst$$reg),
8748 as_Register($src2$$reg),
8749 as_Register($src1$$reg),
8750 (Assembler::Condition)$cmp$$cmpcode);
8751 %}
8752
8753 ins_pipe(icond_reg_reg);
8754 %}
8755
8756 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8757 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8758
8759 ins_cost(INSN_COST * 2);
8760 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8761
8762 ins_encode %{
8763 __ csel(as_Register($dst$$reg),
8764 as_Register($src2$$reg),
8765 as_Register($src1$$reg),
8766 (Assembler::Condition)$cmp$$cmpcode);
8767 %}
8768
8769 ins_pipe(icond_reg_reg);
8770 %}
8771
8772 // special cases where one arg is zero
8773
8774 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8775 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8776
8777 ins_cost(INSN_COST * 2);
8778 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8779
8780 ins_encode %{
8781 __ csel(as_Register($dst$$reg),
8782 zr,
8783 as_Register($src$$reg),
8784 (Assembler::Condition)$cmp$$cmpcode);
8785 %}
8786
8787 ins_pipe(icond_reg);
8788 %}
8789
8790 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8791 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8792
8793 ins_cost(INSN_COST * 2);
8794 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8795
8796 ins_encode %{
8797 __ csel(as_Register($dst$$reg),
8798 zr,
8799 as_Register($src$$reg),
8800 (Assembler::Condition)$cmp$$cmpcode);
8801 %}
8802
8803 ins_pipe(icond_reg);
8804 %}
8805
8806 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8807 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8808
8809 ins_cost(INSN_COST * 2);
8810 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8811
8812 ins_encode %{
8813 __ csel(as_Register($dst$$reg),
8814 as_Register($src$$reg),
8815 zr,
8816 (Assembler::Condition)$cmp$$cmpcode);
8817 %}
8818
8819 ins_pipe(icond_reg);
8820 %}
8821
8822 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8823 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8824
8825 ins_cost(INSN_COST * 2);
8826 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8827
8828 ins_encode %{
8829 __ csel(as_Register($dst$$reg),
8830 as_Register($src$$reg),
8831 zr,
8832 (Assembler::Condition)$cmp$$cmpcode);
8833 %}
8834
8835 ins_pipe(icond_reg);
8836 %}
8837
8838 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8839 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8840
8841 ins_cost(INSN_COST * 2);
8842 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8843
8844 ins_encode %{
8845 __ cselw(as_Register($dst$$reg),
8846 as_Register($src2$$reg),
8847 as_Register($src1$$reg),
8848 (Assembler::Condition)$cmp$$cmpcode);
8849 %}
8850
8851 ins_pipe(icond_reg_reg);
8852 %}
8853
8854 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8855 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8856
8857 ins_cost(INSN_COST * 2);
8858 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8859
8860 ins_encode %{
8861 __ cselw(as_Register($dst$$reg),
8862 as_Register($src2$$reg),
8863 as_Register($src1$$reg),
8864 (Assembler::Condition)$cmp$$cmpcode);
8865 %}
8866
8867 ins_pipe(icond_reg_reg);
8868 %}
8869
8870 // special cases where one arg is zero
8871
8872 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8873 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8874
8875 ins_cost(INSN_COST * 2);
8876 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8877
8878 ins_encode %{
8879 __ cselw(as_Register($dst$$reg),
8880 zr,
8881 as_Register($src$$reg),
8882 (Assembler::Condition)$cmp$$cmpcode);
8883 %}
8884
8885 ins_pipe(icond_reg);
8886 %}
8887
8888 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8889 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8890
8891 ins_cost(INSN_COST * 2);
8892 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8893
8894 ins_encode %{
8895 __ cselw(as_Register($dst$$reg),
8896 zr,
8897 as_Register($src$$reg),
8898 (Assembler::Condition)$cmp$$cmpcode);
8899 %}
8900
8901 ins_pipe(icond_reg);
8902 %}
8903
8904 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8905 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8906
8907 ins_cost(INSN_COST * 2);
8908 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8909
8910 ins_encode %{
8911 __ cselw(as_Register($dst$$reg),
8912 as_Register($src$$reg),
8913 zr,
8914 (Assembler::Condition)$cmp$$cmpcode);
8915 %}
8916
8917 ins_pipe(icond_reg);
8918 %}
8919
8920 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8921 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8922
8923 ins_cost(INSN_COST * 2);
8924 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8925
8926 ins_encode %{
8927 __ cselw(as_Register($dst$$reg),
8928 as_Register($src$$reg),
8929 zr,
8930 (Assembler::Condition)$cmp$$cmpcode);
8931 %}
8932
8933 ins_pipe(icond_reg);
8934 %}
8935
8936 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8937 %{
8938 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8939
8940 ins_cost(INSN_COST * 3);
8941
8942 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8943 ins_encode %{
8944 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8945 __ fcsels(as_FloatRegister($dst$$reg),
8946 as_FloatRegister($src2$$reg),
8947 as_FloatRegister($src1$$reg),
8948 cond);
8949 %}
8950
8951 ins_pipe(fp_cond_reg_reg_s);
8952 %}
8953
8954 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
8955 %{
8956 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8957
8958 ins_cost(INSN_COST * 3);
8959
8960 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8961 ins_encode %{
8962 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8963 __ fcsels(as_FloatRegister($dst$$reg),
8964 as_FloatRegister($src2$$reg),
8965 as_FloatRegister($src1$$reg),
8966 cond);
8967 %}
8968
8969 ins_pipe(fp_cond_reg_reg_s);
8970 %}
8971
8972 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
8973 %{
8974 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8975
8976 ins_cost(INSN_COST * 3);
8977
8978 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8979 ins_encode %{
8980 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8981 __ fcseld(as_FloatRegister($dst$$reg),
8982 as_FloatRegister($src2$$reg),
8983 as_FloatRegister($src1$$reg),
8984 cond);
8985 %}
8986
8987 ins_pipe(fp_cond_reg_reg_d);
8988 %}
8989
8990 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
8991 %{
8992 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8993
8994 ins_cost(INSN_COST * 3);
8995
8996 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8997 ins_encode %{
8998 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8999 __ fcseld(as_FloatRegister($dst$$reg),
9000 as_FloatRegister($src2$$reg),
9001 as_FloatRegister($src1$$reg),
9002 cond);
9003 %}
9004
9005 ins_pipe(fp_cond_reg_reg_d);
9006 %}
9007
9008 // ============================================================================
9009 // Arithmetic Instructions
9010 //
9011
9012 // Integer Addition
9013
9014 // TODO
9015 // these currently employ operations which do not set CR and hence are
9016 // not flagged as killing CR but we would like to isolate the cases
9017 // where we want to set flags from those where we don't. need to work
9018 // out how to do that.
9019
9020 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9021 match(Set dst (AddI src1 src2));
9022
9023 ins_cost(INSN_COST);
9024 format %{ "addw $dst, $src1, $src2" %}
9025
9026 ins_encode %{
9027 __ addw(as_Register($dst$$reg),
9028 as_Register($src1$$reg),
9029 as_Register($src2$$reg));
9030 %}
9031
9032 ins_pipe(ialu_reg_reg);
9033 %}
9034
9035 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9036 match(Set dst (AddI src1 src2));
9037
9038 ins_cost(INSN_COST);
9039 format %{ "addw $dst, $src1, $src2" %}
9040
9041 // use opcode to indicate that this is an add not a sub
9042 opcode(0x0);
9043
9044 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9045
9046 ins_pipe(ialu_reg_imm);
9047 %}
9048
9049 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9050 match(Set dst (AddI (ConvL2I src1) src2));
9051
9052 ins_cost(INSN_COST);
9053 format %{ "addw $dst, $src1, $src2" %}
9054
9055 // use opcode to indicate that this is an add not a sub
9056 opcode(0x0);
9057
9058 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9059
9060 ins_pipe(ialu_reg_imm);
9061 %}
9062
9063 // Pointer Addition
9064 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9065 match(Set dst (AddP src1 src2));
9066
9067 ins_cost(INSN_COST);
9068 format %{ "add $dst, $src1, $src2\t# ptr" %}
9069
9070 ins_encode %{
9071 __ add(as_Register($dst$$reg),
9072 as_Register($src1$$reg),
9073 as_Register($src2$$reg));
9074 %}
9075
9076 ins_pipe(ialu_reg_reg);
9077 %}
9078
9079 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9080 match(Set dst (AddP src1 (ConvI2L src2)));
9081
9082 ins_cost(1.9 * INSN_COST);
9083 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9084
9085 ins_encode %{
9086 __ add(as_Register($dst$$reg),
9087 as_Register($src1$$reg),
9088 as_Register($src2$$reg), ext::sxtw);
9089 %}
9090
9091 ins_pipe(ialu_reg_reg);
9092 %}
9093
9094 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9095 match(Set dst (AddP src1 (LShiftL src2 scale)));
9096
9097 ins_cost(1.9 * INSN_COST);
9098 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9099
9100 ins_encode %{
9101 __ lea(as_Register($dst$$reg),
9102 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9103 Address::lsl($scale$$constant)));
9104 %}
9105
9106 ins_pipe(ialu_reg_reg_shift);
9107 %}
9108
9109 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9110 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9111
9112 ins_cost(1.9 * INSN_COST);
9113 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9114
9115 ins_encode %{
9116 __ lea(as_Register($dst$$reg),
9117 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9118 Address::sxtw($scale$$constant)));
9119 %}
9120
9121 ins_pipe(ialu_reg_reg_shift);
9122 %}
9123
9124 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9125 match(Set dst (LShiftL (ConvI2L src) scale));
9126
9127 ins_cost(INSN_COST);
9128 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9129
9130 ins_encode %{
9131 __ sbfiz(as_Register($dst$$reg),
9132 as_Register($src$$reg),
9133 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9134 %}
9135
9136 ins_pipe(ialu_reg_shift);
9137 %}
9138
9139 // Pointer Immediate Addition
9140 // n.b. this needs to be more expensive than using an indirect memory
9141 // operand
9142 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9143 match(Set dst (AddP src1 src2));
9144
9145 ins_cost(INSN_COST);
9146 format %{ "add $dst, $src1, $src2\t# ptr" %}
9147
9148 // use opcode to indicate that this is an add not a sub
9149 opcode(0x0);
9150
9151 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9152
9153 ins_pipe(ialu_reg_imm);
9154 %}
9155
9156 // Long Addition
9157 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9158
9159 match(Set dst (AddL src1 src2));
9160
9161 ins_cost(INSN_COST);
9162 format %{ "add $dst, $src1, $src2" %}
9163
9164 ins_encode %{
9165 __ add(as_Register($dst$$reg),
9166 as_Register($src1$$reg),
9167 as_Register($src2$$reg));
9168 %}
9169
9170 ins_pipe(ialu_reg_reg);
9171 %}
9172
9173 // No constant pool entries requiredLong Immediate Addition.
9174 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9175 match(Set dst (AddL src1 src2));
9176
9177 ins_cost(INSN_COST);
9178 format %{ "add $dst, $src1, $src2" %}
9179
9180 // use opcode to indicate that this is an add not a sub
9181 opcode(0x0);
9182
9183 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9184
9185 ins_pipe(ialu_reg_imm);
9186 %}
9187
9188 // Integer Subtraction
9189 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9190 match(Set dst (SubI src1 src2));
9191
9192 ins_cost(INSN_COST);
9193 format %{ "subw $dst, $src1, $src2" %}
9194
9195 ins_encode %{
9196 __ subw(as_Register($dst$$reg),
9197 as_Register($src1$$reg),
9198 as_Register($src2$$reg));
9199 %}
9200
9201 ins_pipe(ialu_reg_reg);
9202 %}
9203
9204 // Immediate Subtraction
9205 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9206 match(Set dst (SubI src1 src2));
9207
9208 ins_cost(INSN_COST);
9209 format %{ "subw $dst, $src1, $src2" %}
9210
9211 // use opcode to indicate that this is a sub not an add
9212 opcode(0x1);
9213
9214 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9215
9216 ins_pipe(ialu_reg_imm);
9217 %}
9218
9219 // Long Subtraction
9220 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9221
9222 match(Set dst (SubL src1 src2));
9223
9224 ins_cost(INSN_COST);
9225 format %{ "sub $dst, $src1, $src2" %}
9226
9227 ins_encode %{
9228 __ sub(as_Register($dst$$reg),
9229 as_Register($src1$$reg),
9230 as_Register($src2$$reg));
9231 %}
9232
9233 ins_pipe(ialu_reg_reg);
9234 %}
9235
9236 // No constant pool entries requiredLong Immediate Subtraction.
9237 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9238 match(Set dst (SubL src1 src2));
9239
9240 ins_cost(INSN_COST);
9241 format %{ "sub$dst, $src1, $src2" %}
9242
9243 // use opcode to indicate that this is a sub not an add
9244 opcode(0x1);
9245
9246 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9247
9248 ins_pipe(ialu_reg_imm);
9249 %}
9250
9251 // Integer Negation (special case for sub)
9252
9253 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9254 match(Set dst (SubI zero src));
9255
9256 ins_cost(INSN_COST);
9257 format %{ "negw $dst, $src\t# int" %}
9258
9259 ins_encode %{
9260 __ negw(as_Register($dst$$reg),
9261 as_Register($src$$reg));
9262 %}
9263
9264 ins_pipe(ialu_reg);
9265 %}
9266
9267 // Long Negation
9268
9269 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9270 match(Set dst (SubL zero src));
9271
9272 ins_cost(INSN_COST);
9273 format %{ "neg $dst, $src\t# long" %}
9274
9275 ins_encode %{
9276 __ neg(as_Register($dst$$reg),
9277 as_Register($src$$reg));
9278 %}
9279
9280 ins_pipe(ialu_reg);
9281 %}
9282
9283 // Integer Multiply
9284
9285 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9286 match(Set dst (MulI src1 src2));
9287
9288 ins_cost(INSN_COST * 3);
9289 format %{ "mulw $dst, $src1, $src2" %}
9290
9291 ins_encode %{
9292 __ mulw(as_Register($dst$$reg),
9293 as_Register($src1$$reg),
9294 as_Register($src2$$reg));
9295 %}
9296
9297 ins_pipe(imul_reg_reg);
9298 %}
9299
9300 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9301 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9302
9303 ins_cost(INSN_COST * 3);
9304 format %{ "smull $dst, $src1, $src2" %}
9305
9306 ins_encode %{
9307 __ smull(as_Register($dst$$reg),
9308 as_Register($src1$$reg),
9309 as_Register($src2$$reg));
9310 %}
9311
9312 ins_pipe(imul_reg_reg);
9313 %}
9314
9315 // Long Multiply
9316
9317 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9318 match(Set dst (MulL src1 src2));
9319
9320 ins_cost(INSN_COST * 5);
9321 format %{ "mul $dst, $src1, $src2" %}
9322
9323 ins_encode %{
9324 __ mul(as_Register($dst$$reg),
9325 as_Register($src1$$reg),
9326 as_Register($src2$$reg));
9327 %}
9328
9329 ins_pipe(lmul_reg_reg);
9330 %}
9331
9332 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9333 %{
9334 match(Set dst (MulHiL src1 src2));
9335
9336 ins_cost(INSN_COST * 7);
9337 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9338
9339 ins_encode %{
9340 __ smulh(as_Register($dst$$reg),
9341 as_Register($src1$$reg),
9342 as_Register($src2$$reg));
9343 %}
9344
9345 ins_pipe(lmul_reg_reg);
9346 %}
9347
9348 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9349 %{
9350 match(Set dst (UMulHiL src1 src2));
9351
9352 ins_cost(INSN_COST * 7);
9353 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9354
9355 ins_encode %{
9356 __ umulh(as_Register($dst$$reg),
9357 as_Register($src1$$reg),
9358 as_Register($src2$$reg));
9359 %}
9360
9361 ins_pipe(lmul_reg_reg);
9362 %}
9363
9364 // Combined Integer Multiply & Add/Sub
9365
9366 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9367 match(Set dst (AddI src3 (MulI src1 src2)));
9368
9369 ins_cost(INSN_COST * 3);
9370 format %{ "madd $dst, $src1, $src2, $src3" %}
9371
9372 ins_encode %{
9373 __ maddw(as_Register($dst$$reg),
9374 as_Register($src1$$reg),
9375 as_Register($src2$$reg),
9376 as_Register($src3$$reg));
9377 %}
9378
9379 ins_pipe(imac_reg_reg);
9380 %}
9381
9382 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9383 match(Set dst (SubI src3 (MulI src1 src2)));
9384
9385 ins_cost(INSN_COST * 3);
9386 format %{ "msub $dst, $src1, $src2, $src3" %}
9387
9388 ins_encode %{
9389 __ msubw(as_Register($dst$$reg),
9390 as_Register($src1$$reg),
9391 as_Register($src2$$reg),
9392 as_Register($src3$$reg));
9393 %}
9394
9395 ins_pipe(imac_reg_reg);
9396 %}
9397
9398 // Combined Integer Multiply & Neg
9399
9400 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9401 match(Set dst (MulI (SubI zero src1) src2));
9402
9403 ins_cost(INSN_COST * 3);
9404 format %{ "mneg $dst, $src1, $src2" %}
9405
9406 ins_encode %{
9407 __ mnegw(as_Register($dst$$reg),
9408 as_Register($src1$$reg),
9409 as_Register($src2$$reg));
9410 %}
9411
9412 ins_pipe(imac_reg_reg);
9413 %}
9414
9415 // Combined Long Multiply & Add/Sub
9416
9417 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9418 match(Set dst (AddL src3 (MulL src1 src2)));
9419
9420 ins_cost(INSN_COST * 5);
9421 format %{ "madd $dst, $src1, $src2, $src3" %}
9422
9423 ins_encode %{
9424 __ madd(as_Register($dst$$reg),
9425 as_Register($src1$$reg),
9426 as_Register($src2$$reg),
9427 as_Register($src3$$reg));
9428 %}
9429
9430 ins_pipe(lmac_reg_reg);
9431 %}
9432
9433 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9434 match(Set dst (SubL src3 (MulL src1 src2)));
9435
9436 ins_cost(INSN_COST * 5);
9437 format %{ "msub $dst, $src1, $src2, $src3" %}
9438
9439 ins_encode %{
9440 __ msub(as_Register($dst$$reg),
9441 as_Register($src1$$reg),
9442 as_Register($src2$$reg),
9443 as_Register($src3$$reg));
9444 %}
9445
9446 ins_pipe(lmac_reg_reg);
9447 %}
9448
9449 // Combined Long Multiply & Neg
9450
9451 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9452 match(Set dst (MulL (SubL zero src1) src2));
9453
9454 ins_cost(INSN_COST * 5);
9455 format %{ "mneg $dst, $src1, $src2" %}
9456
9457 ins_encode %{
9458 __ mneg(as_Register($dst$$reg),
9459 as_Register($src1$$reg),
9460 as_Register($src2$$reg));
9461 %}
9462
9463 ins_pipe(lmac_reg_reg);
9464 %}
9465
9466 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9467
9468 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9469 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9470
9471 ins_cost(INSN_COST * 3);
9472 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9473
9474 ins_encode %{
9475 __ smaddl(as_Register($dst$$reg),
9476 as_Register($src1$$reg),
9477 as_Register($src2$$reg),
9478 as_Register($src3$$reg));
9479 %}
9480
9481 ins_pipe(imac_reg_reg);
9482 %}
9483
9484 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9485 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9486
9487 ins_cost(INSN_COST * 3);
9488 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9489
9490 ins_encode %{
9491 __ smsubl(as_Register($dst$$reg),
9492 as_Register($src1$$reg),
9493 as_Register($src2$$reg),
9494 as_Register($src3$$reg));
9495 %}
9496
9497 ins_pipe(imac_reg_reg);
9498 %}
9499
9500 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9501 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9502
9503 ins_cost(INSN_COST * 3);
9504 format %{ "smnegl $dst, $src1, $src2" %}
9505
9506 ins_encode %{
9507 __ smnegl(as_Register($dst$$reg),
9508 as_Register($src1$$reg),
9509 as_Register($src2$$reg));
9510 %}
9511
9512 ins_pipe(imac_reg_reg);
9513 %}
9514
9515 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9516
9517 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9518 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9519
9520 ins_cost(INSN_COST * 5);
9521 format %{ "mulw rscratch1, $src1, $src2\n\t"
9522 "maddw $dst, $src3, $src4, rscratch1" %}
9523
9524 ins_encode %{
9525 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9526 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9527
9528 ins_pipe(imac_reg_reg);
9529 %}
9530
9531 // Integer Divide
9532
9533 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9534 match(Set dst (DivI src1 src2));
9535
9536 ins_cost(INSN_COST * 19);
9537 format %{ "sdivw $dst, $src1, $src2" %}
9538
9539 ins_encode(aarch64_enc_divw(dst, src1, src2));
9540 ins_pipe(idiv_reg_reg);
9541 %}
9542
9543 // Long Divide
9544
9545 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9546 match(Set dst (DivL src1 src2));
9547
9548 ins_cost(INSN_COST * 35);
9549 format %{ "sdiv $dst, $src1, $src2" %}
9550
9551 ins_encode(aarch64_enc_div(dst, src1, src2));
9552 ins_pipe(ldiv_reg_reg);
9553 %}
9554
9555 // Integer Remainder
9556
9557 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9558 match(Set dst (ModI src1 src2));
9559
9560 ins_cost(INSN_COST * 22);
9561 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9562 "msubw $dst, rscratch1, $src2, $src1" %}
9563
9564 ins_encode(aarch64_enc_modw(dst, src1, src2));
9565 ins_pipe(idiv_reg_reg);
9566 %}
9567
9568 // Long Remainder
9569
9570 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9571 match(Set dst (ModL src1 src2));
9572
9573 ins_cost(INSN_COST * 38);
9574 format %{ "sdiv rscratch1, $src1, $src2\n"
9575 "msub $dst, rscratch1, $src2, $src1" %}
9576
9577 ins_encode(aarch64_enc_mod(dst, src1, src2));
9578 ins_pipe(ldiv_reg_reg);
9579 %}
9580
9581 // Unsigned Integer Divide
9582
9583 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9584 match(Set dst (UDivI src1 src2));
9585
9586 ins_cost(INSN_COST * 19);
9587 format %{ "udivw $dst, $src1, $src2" %}
9588
9589 ins_encode %{
9590 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9591 %}
9592
9593 ins_pipe(idiv_reg_reg);
9594 %}
9595
9596 // Unsigned Long Divide
9597
9598 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9599 match(Set dst (UDivL src1 src2));
9600
9601 ins_cost(INSN_COST * 35);
9602 format %{ "udiv $dst, $src1, $src2" %}
9603
9604 ins_encode %{
9605 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9606 %}
9607
9608 ins_pipe(ldiv_reg_reg);
9609 %}
9610
9611 // Unsigned Integer Remainder
9612
9613 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9614 match(Set dst (UModI src1 src2));
9615
9616 ins_cost(INSN_COST * 22);
9617 format %{ "udivw rscratch1, $src1, $src2\n\t"
9618 "msubw $dst, rscratch1, $src2, $src1" %}
9619
9620 ins_encode %{
9621 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9622 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9623 %}
9624
9625 ins_pipe(idiv_reg_reg);
9626 %}
9627
9628 // Unsigned Long Remainder
9629
9630 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9631 match(Set dst (UModL src1 src2));
9632
9633 ins_cost(INSN_COST * 38);
9634 format %{ "udiv rscratch1, $src1, $src2\n"
9635 "msub $dst, rscratch1, $src2, $src1" %}
9636
9637 ins_encode %{
9638 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9639 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9640 %}
9641
9642 ins_pipe(ldiv_reg_reg);
9643 %}
9644
9645 // Integer Shifts
9646
9647 // Shift Left Register
9648 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9649 match(Set dst (LShiftI src1 src2));
9650
9651 ins_cost(INSN_COST * 2);
9652 format %{ "lslvw $dst, $src1, $src2" %}
9653
9654 ins_encode %{
9655 __ lslvw(as_Register($dst$$reg),
9656 as_Register($src1$$reg),
9657 as_Register($src2$$reg));
9658 %}
9659
9660 ins_pipe(ialu_reg_reg_vshift);
9661 %}
9662
9663 // Shift Left Immediate
9664 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9665 match(Set dst (LShiftI src1 src2));
9666
9667 ins_cost(INSN_COST);
9668 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9669
9670 ins_encode %{
9671 __ lslw(as_Register($dst$$reg),
9672 as_Register($src1$$reg),
9673 $src2$$constant & 0x1f);
9674 %}
9675
9676 ins_pipe(ialu_reg_shift);
9677 %}
9678
9679 // Shift Right Logical Register
9680 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9681 match(Set dst (URShiftI src1 src2));
9682
9683 ins_cost(INSN_COST * 2);
9684 format %{ "lsrvw $dst, $src1, $src2" %}
9685
9686 ins_encode %{
9687 __ lsrvw(as_Register($dst$$reg),
9688 as_Register($src1$$reg),
9689 as_Register($src2$$reg));
9690 %}
9691
9692 ins_pipe(ialu_reg_reg_vshift);
9693 %}
9694
9695 // Shift Right Logical Immediate
9696 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9697 match(Set dst (URShiftI src1 src2));
9698
9699 ins_cost(INSN_COST);
9700 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9701
9702 ins_encode %{
9703 __ lsrw(as_Register($dst$$reg),
9704 as_Register($src1$$reg),
9705 $src2$$constant & 0x1f);
9706 %}
9707
9708 ins_pipe(ialu_reg_shift);
9709 %}
9710
9711 // Shift Right Arithmetic Register
9712 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9713 match(Set dst (RShiftI src1 src2));
9714
9715 ins_cost(INSN_COST * 2);
9716 format %{ "asrvw $dst, $src1, $src2" %}
9717
9718 ins_encode %{
9719 __ asrvw(as_Register($dst$$reg),
9720 as_Register($src1$$reg),
9721 as_Register($src2$$reg));
9722 %}
9723
9724 ins_pipe(ialu_reg_reg_vshift);
9725 %}
9726
9727 // Shift Right Arithmetic Immediate
9728 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9729 match(Set dst (RShiftI src1 src2));
9730
9731 ins_cost(INSN_COST);
9732 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9733
9734 ins_encode %{
9735 __ asrw(as_Register($dst$$reg),
9736 as_Register($src1$$reg),
9737 $src2$$constant & 0x1f);
9738 %}
9739
9740 ins_pipe(ialu_reg_shift);
9741 %}
9742
9743 // Combined Int Mask and Right Shift (using UBFM)
9744 // TODO
9745
9746 // Long Shifts
9747
9748 // Shift Left Register
9749 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9750 match(Set dst (LShiftL src1 src2));
9751
9752 ins_cost(INSN_COST * 2);
9753 format %{ "lslv $dst, $src1, $src2" %}
9754
9755 ins_encode %{
9756 __ lslv(as_Register($dst$$reg),
9757 as_Register($src1$$reg),
9758 as_Register($src2$$reg));
9759 %}
9760
9761 ins_pipe(ialu_reg_reg_vshift);
9762 %}
9763
9764 // Shift Left Immediate
9765 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9766 match(Set dst (LShiftL src1 src2));
9767
9768 ins_cost(INSN_COST);
9769 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9770
9771 ins_encode %{
9772 __ lsl(as_Register($dst$$reg),
9773 as_Register($src1$$reg),
9774 $src2$$constant & 0x3f);
9775 %}
9776
9777 ins_pipe(ialu_reg_shift);
9778 %}
9779
9780 // Shift Right Logical Register
9781 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9782 match(Set dst (URShiftL src1 src2));
9783
9784 ins_cost(INSN_COST * 2);
9785 format %{ "lsrv $dst, $src1, $src2" %}
9786
9787 ins_encode %{
9788 __ lsrv(as_Register($dst$$reg),
9789 as_Register($src1$$reg),
9790 as_Register($src2$$reg));
9791 %}
9792
9793 ins_pipe(ialu_reg_reg_vshift);
9794 %}
9795
9796 // Shift Right Logical Immediate
9797 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9798 match(Set dst (URShiftL src1 src2));
9799
9800 ins_cost(INSN_COST);
9801 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9802
9803 ins_encode %{
9804 __ lsr(as_Register($dst$$reg),
9805 as_Register($src1$$reg),
9806 $src2$$constant & 0x3f);
9807 %}
9808
9809 ins_pipe(ialu_reg_shift);
9810 %}
9811
9812 // A special-case pattern for card table stores.
9813 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9814 match(Set dst (URShiftL (CastP2X src1) src2));
9815
9816 ins_cost(INSN_COST);
9817 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9818
9819 ins_encode %{
9820 __ lsr(as_Register($dst$$reg),
9821 as_Register($src1$$reg),
9822 $src2$$constant & 0x3f);
9823 %}
9824
9825 ins_pipe(ialu_reg_shift);
9826 %}
9827
9828 // Shift Right Arithmetic Register
9829 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9830 match(Set dst (RShiftL src1 src2));
9831
9832 ins_cost(INSN_COST * 2);
9833 format %{ "asrv $dst, $src1, $src2" %}
9834
9835 ins_encode %{
9836 __ asrv(as_Register($dst$$reg),
9837 as_Register($src1$$reg),
9838 as_Register($src2$$reg));
9839 %}
9840
9841 ins_pipe(ialu_reg_reg_vshift);
9842 %}
9843
9844 // Shift Right Arithmetic Immediate
9845 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9846 match(Set dst (RShiftL src1 src2));
9847
9848 ins_cost(INSN_COST);
9849 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9850
9851 ins_encode %{
9852 __ asr(as_Register($dst$$reg),
9853 as_Register($src1$$reg),
9854 $src2$$constant & 0x3f);
9855 %}
9856
9857 ins_pipe(ialu_reg_shift);
9858 %}
9859
9860 // BEGIN This section of the file is automatically generated. Do not edit --------------
9861 // This section is generated from aarch64_ad.m4
9862
9863 // This pattern is automatically generated from aarch64_ad.m4
9864 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9865 instruct regL_not_reg(iRegLNoSp dst,
9866 iRegL src1, immL_M1 m1,
9867 rFlagsReg cr) %{
9868 match(Set dst (XorL src1 m1));
9869 ins_cost(INSN_COST);
9870 format %{ "eon $dst, $src1, zr" %}
9871
9872 ins_encode %{
9873 __ eon(as_Register($dst$$reg),
9874 as_Register($src1$$reg),
9875 zr,
9876 Assembler::LSL, 0);
9877 %}
9878
9879 ins_pipe(ialu_reg);
9880 %}
9881
9882 // This pattern is automatically generated from aarch64_ad.m4
9883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9884 instruct regI_not_reg(iRegINoSp dst,
9885 iRegIorL2I src1, immI_M1 m1,
9886 rFlagsReg cr) %{
9887 match(Set dst (XorI src1 m1));
9888 ins_cost(INSN_COST);
9889 format %{ "eonw $dst, $src1, zr" %}
9890
9891 ins_encode %{
9892 __ eonw(as_Register($dst$$reg),
9893 as_Register($src1$$reg),
9894 zr,
9895 Assembler::LSL, 0);
9896 %}
9897
9898 ins_pipe(ialu_reg);
9899 %}
9900
9901 // This pattern is automatically generated from aarch64_ad.m4
9902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9903 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9904 immI0 zero, iRegIorL2I src1, immI src2) %{
9905 match(Set dst (SubI zero (URShiftI src1 src2)));
9906
9907 ins_cost(1.9 * INSN_COST);
9908 format %{ "negw $dst, $src1, LSR $src2" %}
9909
9910 ins_encode %{
9911 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9912 Assembler::LSR, $src2$$constant & 0x1f);
9913 %}
9914
9915 ins_pipe(ialu_reg_shift);
9916 %}
9917
9918 // This pattern is automatically generated from aarch64_ad.m4
9919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9920 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9921 immI0 zero, iRegIorL2I src1, immI src2) %{
9922 match(Set dst (SubI zero (RShiftI src1 src2)));
9923
9924 ins_cost(1.9 * INSN_COST);
9925 format %{ "negw $dst, $src1, ASR $src2" %}
9926
9927 ins_encode %{
9928 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9929 Assembler::ASR, $src2$$constant & 0x1f);
9930 %}
9931
9932 ins_pipe(ialu_reg_shift);
9933 %}
9934
9935 // This pattern is automatically generated from aarch64_ad.m4
9936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9937 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9938 immI0 zero, iRegIorL2I src1, immI src2) %{
9939 match(Set dst (SubI zero (LShiftI src1 src2)));
9940
9941 ins_cost(1.9 * INSN_COST);
9942 format %{ "negw $dst, $src1, LSL $src2" %}
9943
9944 ins_encode %{
9945 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9946 Assembler::LSL, $src2$$constant & 0x1f);
9947 %}
9948
9949 ins_pipe(ialu_reg_shift);
9950 %}
9951
9952 // This pattern is automatically generated from aarch64_ad.m4
9953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9954 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
9955 immL0 zero, iRegL src1, immI src2) %{
9956 match(Set dst (SubL zero (URShiftL src1 src2)));
9957
9958 ins_cost(1.9 * INSN_COST);
9959 format %{ "neg $dst, $src1, LSR $src2" %}
9960
9961 ins_encode %{
9962 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9963 Assembler::LSR, $src2$$constant & 0x3f);
9964 %}
9965
9966 ins_pipe(ialu_reg_shift);
9967 %}
9968
9969 // This pattern is automatically generated from aarch64_ad.m4
9970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9971 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
9972 immL0 zero, iRegL src1, immI src2) %{
9973 match(Set dst (SubL zero (RShiftL src1 src2)));
9974
9975 ins_cost(1.9 * INSN_COST);
9976 format %{ "neg $dst, $src1, ASR $src2" %}
9977
9978 ins_encode %{
9979 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9980 Assembler::ASR, $src2$$constant & 0x3f);
9981 %}
9982
9983 ins_pipe(ialu_reg_shift);
9984 %}
9985
9986 // This pattern is automatically generated from aarch64_ad.m4
9987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9988 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
9989 immL0 zero, iRegL src1, immI src2) %{
9990 match(Set dst (SubL zero (LShiftL src1 src2)));
9991
9992 ins_cost(1.9 * INSN_COST);
9993 format %{ "neg $dst, $src1, LSL $src2" %}
9994
9995 ins_encode %{
9996 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9997 Assembler::LSL, $src2$$constant & 0x3f);
9998 %}
9999
10000 ins_pipe(ialu_reg_shift);
10001 %}
10002
10003 // This pattern is automatically generated from aarch64_ad.m4
10004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10005 instruct AndI_reg_not_reg(iRegINoSp dst,
10006 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10007 match(Set dst (AndI src1 (XorI src2 m1)));
10008 ins_cost(INSN_COST);
10009 format %{ "bicw $dst, $src1, $src2" %}
10010
10011 ins_encode %{
10012 __ bicw(as_Register($dst$$reg),
10013 as_Register($src1$$reg),
10014 as_Register($src2$$reg),
10015 Assembler::LSL, 0);
10016 %}
10017
10018 ins_pipe(ialu_reg_reg);
10019 %}
10020
10021 // This pattern is automatically generated from aarch64_ad.m4
10022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10023 instruct AndL_reg_not_reg(iRegLNoSp dst,
10024 iRegL src1, iRegL src2, immL_M1 m1) %{
10025 match(Set dst (AndL src1 (XorL src2 m1)));
10026 ins_cost(INSN_COST);
10027 format %{ "bic $dst, $src1, $src2" %}
10028
10029 ins_encode %{
10030 __ bic(as_Register($dst$$reg),
10031 as_Register($src1$$reg),
10032 as_Register($src2$$reg),
10033 Assembler::LSL, 0);
10034 %}
10035
10036 ins_pipe(ialu_reg_reg);
10037 %}
10038
10039 // This pattern is automatically generated from aarch64_ad.m4
10040 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10041 instruct OrI_reg_not_reg(iRegINoSp dst,
10042 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10043 match(Set dst (OrI src1 (XorI src2 m1)));
10044 ins_cost(INSN_COST);
10045 format %{ "ornw $dst, $src1, $src2" %}
10046
10047 ins_encode %{
10048 __ ornw(as_Register($dst$$reg),
10049 as_Register($src1$$reg),
10050 as_Register($src2$$reg),
10051 Assembler::LSL, 0);
10052 %}
10053
10054 ins_pipe(ialu_reg_reg);
10055 %}
10056
10057 // This pattern is automatically generated from aarch64_ad.m4
10058 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10059 instruct OrL_reg_not_reg(iRegLNoSp dst,
10060 iRegL src1, iRegL src2, immL_M1 m1) %{
10061 match(Set dst (OrL src1 (XorL src2 m1)));
10062 ins_cost(INSN_COST);
10063 format %{ "orn $dst, $src1, $src2" %}
10064
10065 ins_encode %{
10066 __ orn(as_Register($dst$$reg),
10067 as_Register($src1$$reg),
10068 as_Register($src2$$reg),
10069 Assembler::LSL, 0);
10070 %}
10071
10072 ins_pipe(ialu_reg_reg);
10073 %}
10074
10075 // This pattern is automatically generated from aarch64_ad.m4
10076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10077 instruct XorI_reg_not_reg(iRegINoSp dst,
10078 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10079 match(Set dst (XorI m1 (XorI src2 src1)));
10080 ins_cost(INSN_COST);
10081 format %{ "eonw $dst, $src1, $src2" %}
10082
10083 ins_encode %{
10084 __ eonw(as_Register($dst$$reg),
10085 as_Register($src1$$reg),
10086 as_Register($src2$$reg),
10087 Assembler::LSL, 0);
10088 %}
10089
10090 ins_pipe(ialu_reg_reg);
10091 %}
10092
10093 // This pattern is automatically generated from aarch64_ad.m4
10094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10095 instruct XorL_reg_not_reg(iRegLNoSp dst,
10096 iRegL src1, iRegL src2, immL_M1 m1) %{
10097 match(Set dst (XorL m1 (XorL src2 src1)));
10098 ins_cost(INSN_COST);
10099 format %{ "eon $dst, $src1, $src2" %}
10100
10101 ins_encode %{
10102 __ eon(as_Register($dst$$reg),
10103 as_Register($src1$$reg),
10104 as_Register($src2$$reg),
10105 Assembler::LSL, 0);
10106 %}
10107
10108 ins_pipe(ialu_reg_reg);
10109 %}
10110
10111 // This pattern is automatically generated from aarch64_ad.m4
10112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10113 // val & (-1 ^ (val >>> shift)) ==> bicw
10114 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10115 iRegIorL2I src1, iRegIorL2I src2,
10116 immI src3, immI_M1 src4) %{
10117 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10118 ins_cost(1.9 * INSN_COST);
10119 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10120
10121 ins_encode %{
10122 __ bicw(as_Register($dst$$reg),
10123 as_Register($src1$$reg),
10124 as_Register($src2$$reg),
10125 Assembler::LSR,
10126 $src3$$constant & 0x1f);
10127 %}
10128
10129 ins_pipe(ialu_reg_reg_shift);
10130 %}
10131
10132 // This pattern is automatically generated from aarch64_ad.m4
10133 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10134 // val & (-1 ^ (val >>> shift)) ==> bic
10135 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10136 iRegL src1, iRegL src2,
10137 immI src3, immL_M1 src4) %{
10138 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10139 ins_cost(1.9 * INSN_COST);
10140 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10141
10142 ins_encode %{
10143 __ bic(as_Register($dst$$reg),
10144 as_Register($src1$$reg),
10145 as_Register($src2$$reg),
10146 Assembler::LSR,
10147 $src3$$constant & 0x3f);
10148 %}
10149
10150 ins_pipe(ialu_reg_reg_shift);
10151 %}
10152
10153 // This pattern is automatically generated from aarch64_ad.m4
10154 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10155 // val & (-1 ^ (val >> shift)) ==> bicw
10156 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10157 iRegIorL2I src1, iRegIorL2I src2,
10158 immI src3, immI_M1 src4) %{
10159 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10160 ins_cost(1.9 * INSN_COST);
10161 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10162
10163 ins_encode %{
10164 __ bicw(as_Register($dst$$reg),
10165 as_Register($src1$$reg),
10166 as_Register($src2$$reg),
10167 Assembler::ASR,
10168 $src3$$constant & 0x1f);
10169 %}
10170
10171 ins_pipe(ialu_reg_reg_shift);
10172 %}
10173
10174 // This pattern is automatically generated from aarch64_ad.m4
10175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10176 // val & (-1 ^ (val >> shift)) ==> bic
10177 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10178 iRegL src1, iRegL src2,
10179 immI src3, immL_M1 src4) %{
10180 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10181 ins_cost(1.9 * INSN_COST);
10182 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10183
10184 ins_encode %{
10185 __ bic(as_Register($dst$$reg),
10186 as_Register($src1$$reg),
10187 as_Register($src2$$reg),
10188 Assembler::ASR,
10189 $src3$$constant & 0x3f);
10190 %}
10191
10192 ins_pipe(ialu_reg_reg_shift);
10193 %}
10194
10195 // This pattern is automatically generated from aarch64_ad.m4
10196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10197 // val & (-1 ^ (val ror shift)) ==> bicw
10198 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10199 iRegIorL2I src1, iRegIorL2I src2,
10200 immI src3, immI_M1 src4) %{
10201 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10202 ins_cost(1.9 * INSN_COST);
10203 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10204
10205 ins_encode %{
10206 __ bicw(as_Register($dst$$reg),
10207 as_Register($src1$$reg),
10208 as_Register($src2$$reg),
10209 Assembler::ROR,
10210 $src3$$constant & 0x1f);
10211 %}
10212
10213 ins_pipe(ialu_reg_reg_shift);
10214 %}
10215
10216 // This pattern is automatically generated from aarch64_ad.m4
10217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10218 // val & (-1 ^ (val ror shift)) ==> bic
10219 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10220 iRegL src1, iRegL src2,
10221 immI src3, immL_M1 src4) %{
10222 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10223 ins_cost(1.9 * INSN_COST);
10224 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10225
10226 ins_encode %{
10227 __ bic(as_Register($dst$$reg),
10228 as_Register($src1$$reg),
10229 as_Register($src2$$reg),
10230 Assembler::ROR,
10231 $src3$$constant & 0x3f);
10232 %}
10233
10234 ins_pipe(ialu_reg_reg_shift);
10235 %}
10236
10237 // This pattern is automatically generated from aarch64_ad.m4
10238 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10239 // val & (-1 ^ (val << shift)) ==> bicw
10240 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10241 iRegIorL2I src1, iRegIorL2I src2,
10242 immI src3, immI_M1 src4) %{
10243 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10244 ins_cost(1.9 * INSN_COST);
10245 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10246
10247 ins_encode %{
10248 __ bicw(as_Register($dst$$reg),
10249 as_Register($src1$$reg),
10250 as_Register($src2$$reg),
10251 Assembler::LSL,
10252 $src3$$constant & 0x1f);
10253 %}
10254
10255 ins_pipe(ialu_reg_reg_shift);
10256 %}
10257
10258 // This pattern is automatically generated from aarch64_ad.m4
10259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10260 // val & (-1 ^ (val << shift)) ==> bic
10261 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10262 iRegL src1, iRegL src2,
10263 immI src3, immL_M1 src4) %{
10264 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10265 ins_cost(1.9 * INSN_COST);
10266 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10267
10268 ins_encode %{
10269 __ bic(as_Register($dst$$reg),
10270 as_Register($src1$$reg),
10271 as_Register($src2$$reg),
10272 Assembler::LSL,
10273 $src3$$constant & 0x3f);
10274 %}
10275
10276 ins_pipe(ialu_reg_reg_shift);
10277 %}
10278
10279 // This pattern is automatically generated from aarch64_ad.m4
10280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10281 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10282 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10283 iRegIorL2I src1, iRegIorL2I src2,
10284 immI src3, immI_M1 src4) %{
10285 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10286 ins_cost(1.9 * INSN_COST);
10287 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10288
10289 ins_encode %{
10290 __ eonw(as_Register($dst$$reg),
10291 as_Register($src1$$reg),
10292 as_Register($src2$$reg),
10293 Assembler::LSR,
10294 $src3$$constant & 0x1f);
10295 %}
10296
10297 ins_pipe(ialu_reg_reg_shift);
10298 %}
10299
10300 // This pattern is automatically generated from aarch64_ad.m4
10301 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10302 // val ^ (-1 ^ (val >>> shift)) ==> eon
10303 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10304 iRegL src1, iRegL src2,
10305 immI src3, immL_M1 src4) %{
10306 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10307 ins_cost(1.9 * INSN_COST);
10308 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10309
10310 ins_encode %{
10311 __ eon(as_Register($dst$$reg),
10312 as_Register($src1$$reg),
10313 as_Register($src2$$reg),
10314 Assembler::LSR,
10315 $src3$$constant & 0x3f);
10316 %}
10317
10318 ins_pipe(ialu_reg_reg_shift);
10319 %}
10320
10321 // This pattern is automatically generated from aarch64_ad.m4
10322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10323 // val ^ (-1 ^ (val >> shift)) ==> eonw
10324 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10325 iRegIorL2I src1, iRegIorL2I src2,
10326 immI src3, immI_M1 src4) %{
10327 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10328 ins_cost(1.9 * INSN_COST);
10329 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10330
10331 ins_encode %{
10332 __ eonw(as_Register($dst$$reg),
10333 as_Register($src1$$reg),
10334 as_Register($src2$$reg),
10335 Assembler::ASR,
10336 $src3$$constant & 0x1f);
10337 %}
10338
10339 ins_pipe(ialu_reg_reg_shift);
10340 %}
10341
10342 // This pattern is automatically generated from aarch64_ad.m4
10343 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10344 // val ^ (-1 ^ (val >> shift)) ==> eon
10345 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10346 iRegL src1, iRegL src2,
10347 immI src3, immL_M1 src4) %{
10348 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10349 ins_cost(1.9 * INSN_COST);
10350 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10351
10352 ins_encode %{
10353 __ eon(as_Register($dst$$reg),
10354 as_Register($src1$$reg),
10355 as_Register($src2$$reg),
10356 Assembler::ASR,
10357 $src3$$constant & 0x3f);
10358 %}
10359
10360 ins_pipe(ialu_reg_reg_shift);
10361 %}
10362
10363 // This pattern is automatically generated from aarch64_ad.m4
10364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10365 // val ^ (-1 ^ (val ror shift)) ==> eonw
10366 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10367 iRegIorL2I src1, iRegIorL2I src2,
10368 immI src3, immI_M1 src4) %{
10369 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10370 ins_cost(1.9 * INSN_COST);
10371 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10372
10373 ins_encode %{
10374 __ eonw(as_Register($dst$$reg),
10375 as_Register($src1$$reg),
10376 as_Register($src2$$reg),
10377 Assembler::ROR,
10378 $src3$$constant & 0x1f);
10379 %}
10380
10381 ins_pipe(ialu_reg_reg_shift);
10382 %}
10383
10384 // This pattern is automatically generated from aarch64_ad.m4
10385 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10386 // val ^ (-1 ^ (val ror shift)) ==> eon
10387 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10388 iRegL src1, iRegL src2,
10389 immI src3, immL_M1 src4) %{
10390 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10391 ins_cost(1.9 * INSN_COST);
10392 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10393
10394 ins_encode %{
10395 __ eon(as_Register($dst$$reg),
10396 as_Register($src1$$reg),
10397 as_Register($src2$$reg),
10398 Assembler::ROR,
10399 $src3$$constant & 0x3f);
10400 %}
10401
10402 ins_pipe(ialu_reg_reg_shift);
10403 %}
10404
10405 // This pattern is automatically generated from aarch64_ad.m4
10406 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10407 // val ^ (-1 ^ (val << shift)) ==> eonw
10408 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10409 iRegIorL2I src1, iRegIorL2I src2,
10410 immI src3, immI_M1 src4) %{
10411 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10412 ins_cost(1.9 * INSN_COST);
10413 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10414
10415 ins_encode %{
10416 __ eonw(as_Register($dst$$reg),
10417 as_Register($src1$$reg),
10418 as_Register($src2$$reg),
10419 Assembler::LSL,
10420 $src3$$constant & 0x1f);
10421 %}
10422
10423 ins_pipe(ialu_reg_reg_shift);
10424 %}
10425
10426 // This pattern is automatically generated from aarch64_ad.m4
10427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10428 // val ^ (-1 ^ (val << shift)) ==> eon
10429 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10430 iRegL src1, iRegL src2,
10431 immI src3, immL_M1 src4) %{
10432 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10433 ins_cost(1.9 * INSN_COST);
10434 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10435
10436 ins_encode %{
10437 __ eon(as_Register($dst$$reg),
10438 as_Register($src1$$reg),
10439 as_Register($src2$$reg),
10440 Assembler::LSL,
10441 $src3$$constant & 0x3f);
10442 %}
10443
10444 ins_pipe(ialu_reg_reg_shift);
10445 %}
10446
10447 // This pattern is automatically generated from aarch64_ad.m4
10448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10449 // val | (-1 ^ (val >>> shift)) ==> ornw
10450 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10451 iRegIorL2I src1, iRegIorL2I src2,
10452 immI src3, immI_M1 src4) %{
10453 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10454 ins_cost(1.9 * INSN_COST);
10455 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10456
10457 ins_encode %{
10458 __ ornw(as_Register($dst$$reg),
10459 as_Register($src1$$reg),
10460 as_Register($src2$$reg),
10461 Assembler::LSR,
10462 $src3$$constant & 0x1f);
10463 %}
10464
10465 ins_pipe(ialu_reg_reg_shift);
10466 %}
10467
10468 // This pattern is automatically generated from aarch64_ad.m4
10469 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10470 // val | (-1 ^ (val >>> shift)) ==> orn
10471 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10472 iRegL src1, iRegL src2,
10473 immI src3, immL_M1 src4) %{
10474 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10475 ins_cost(1.9 * INSN_COST);
10476 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10477
10478 ins_encode %{
10479 __ orn(as_Register($dst$$reg),
10480 as_Register($src1$$reg),
10481 as_Register($src2$$reg),
10482 Assembler::LSR,
10483 $src3$$constant & 0x3f);
10484 %}
10485
10486 ins_pipe(ialu_reg_reg_shift);
10487 %}
10488
10489 // This pattern is automatically generated from aarch64_ad.m4
10490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10491 // val | (-1 ^ (val >> shift)) ==> ornw
10492 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10493 iRegIorL2I src1, iRegIorL2I src2,
10494 immI src3, immI_M1 src4) %{
10495 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10496 ins_cost(1.9 * INSN_COST);
10497 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10498
10499 ins_encode %{
10500 __ ornw(as_Register($dst$$reg),
10501 as_Register($src1$$reg),
10502 as_Register($src2$$reg),
10503 Assembler::ASR,
10504 $src3$$constant & 0x1f);
10505 %}
10506
10507 ins_pipe(ialu_reg_reg_shift);
10508 %}
10509
10510 // This pattern is automatically generated from aarch64_ad.m4
10511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10512 // val | (-1 ^ (val >> shift)) ==> orn
10513 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10514 iRegL src1, iRegL src2,
10515 immI src3, immL_M1 src4) %{
10516 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10517 ins_cost(1.9 * INSN_COST);
10518 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10519
10520 ins_encode %{
10521 __ orn(as_Register($dst$$reg),
10522 as_Register($src1$$reg),
10523 as_Register($src2$$reg),
10524 Assembler::ASR,
10525 $src3$$constant & 0x3f);
10526 %}
10527
10528 ins_pipe(ialu_reg_reg_shift);
10529 %}
10530
10531 // This pattern is automatically generated from aarch64_ad.m4
10532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10533 // val | (-1 ^ (val ror shift)) ==> ornw
10534 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10535 iRegIorL2I src1, iRegIorL2I src2,
10536 immI src3, immI_M1 src4) %{
10537 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10538 ins_cost(1.9 * INSN_COST);
10539 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10540
10541 ins_encode %{
10542 __ ornw(as_Register($dst$$reg),
10543 as_Register($src1$$reg),
10544 as_Register($src2$$reg),
10545 Assembler::ROR,
10546 $src3$$constant & 0x1f);
10547 %}
10548
10549 ins_pipe(ialu_reg_reg_shift);
10550 %}
10551
10552 // This pattern is automatically generated from aarch64_ad.m4
10553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10554 // val | (-1 ^ (val ror shift)) ==> orn
10555 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10556 iRegL src1, iRegL src2,
10557 immI src3, immL_M1 src4) %{
10558 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10559 ins_cost(1.9 * INSN_COST);
10560 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10561
10562 ins_encode %{
10563 __ orn(as_Register($dst$$reg),
10564 as_Register($src1$$reg),
10565 as_Register($src2$$reg),
10566 Assembler::ROR,
10567 $src3$$constant & 0x3f);
10568 %}
10569
10570 ins_pipe(ialu_reg_reg_shift);
10571 %}
10572
10573 // This pattern is automatically generated from aarch64_ad.m4
10574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10575 // val | (-1 ^ (val << shift)) ==> ornw
10576 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10577 iRegIorL2I src1, iRegIorL2I src2,
10578 immI src3, immI_M1 src4) %{
10579 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10580 ins_cost(1.9 * INSN_COST);
10581 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10582
10583 ins_encode %{
10584 __ ornw(as_Register($dst$$reg),
10585 as_Register($src1$$reg),
10586 as_Register($src2$$reg),
10587 Assembler::LSL,
10588 $src3$$constant & 0x1f);
10589 %}
10590
10591 ins_pipe(ialu_reg_reg_shift);
10592 %}
10593
10594 // This pattern is automatically generated from aarch64_ad.m4
10595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10596 // val | (-1 ^ (val << shift)) ==> orn
10597 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10598 iRegL src1, iRegL src2,
10599 immI src3, immL_M1 src4) %{
10600 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10601 ins_cost(1.9 * INSN_COST);
10602 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10603
10604 ins_encode %{
10605 __ orn(as_Register($dst$$reg),
10606 as_Register($src1$$reg),
10607 as_Register($src2$$reg),
10608 Assembler::LSL,
10609 $src3$$constant & 0x3f);
10610 %}
10611
10612 ins_pipe(ialu_reg_reg_shift);
10613 %}
10614
10615 // This pattern is automatically generated from aarch64_ad.m4
10616 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10617 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10618 iRegIorL2I src1, iRegIorL2I src2,
10619 immI src3) %{
10620 match(Set dst (AndI src1 (URShiftI src2 src3)));
10621
10622 ins_cost(1.9 * INSN_COST);
10623 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10624
10625 ins_encode %{
10626 __ andw(as_Register($dst$$reg),
10627 as_Register($src1$$reg),
10628 as_Register($src2$$reg),
10629 Assembler::LSR,
10630 $src3$$constant & 0x1f);
10631 %}
10632
10633 ins_pipe(ialu_reg_reg_shift);
10634 %}
10635
10636 // This pattern is automatically generated from aarch64_ad.m4
10637 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10638 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10639 iRegL src1, iRegL src2,
10640 immI src3) %{
10641 match(Set dst (AndL src1 (URShiftL src2 src3)));
10642
10643 ins_cost(1.9 * INSN_COST);
10644 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10645
10646 ins_encode %{
10647 __ andr(as_Register($dst$$reg),
10648 as_Register($src1$$reg),
10649 as_Register($src2$$reg),
10650 Assembler::LSR,
10651 $src3$$constant & 0x3f);
10652 %}
10653
10654 ins_pipe(ialu_reg_reg_shift);
10655 %}
10656
10657 // This pattern is automatically generated from aarch64_ad.m4
10658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10659 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10660 iRegIorL2I src1, iRegIorL2I src2,
10661 immI src3) %{
10662 match(Set dst (AndI src1 (RShiftI src2 src3)));
10663
10664 ins_cost(1.9 * INSN_COST);
10665 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10666
10667 ins_encode %{
10668 __ andw(as_Register($dst$$reg),
10669 as_Register($src1$$reg),
10670 as_Register($src2$$reg),
10671 Assembler::ASR,
10672 $src3$$constant & 0x1f);
10673 %}
10674
10675 ins_pipe(ialu_reg_reg_shift);
10676 %}
10677
10678 // This pattern is automatically generated from aarch64_ad.m4
10679 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10680 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10681 iRegL src1, iRegL src2,
10682 immI src3) %{
10683 match(Set dst (AndL src1 (RShiftL src2 src3)));
10684
10685 ins_cost(1.9 * INSN_COST);
10686 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10687
10688 ins_encode %{
10689 __ andr(as_Register($dst$$reg),
10690 as_Register($src1$$reg),
10691 as_Register($src2$$reg),
10692 Assembler::ASR,
10693 $src3$$constant & 0x3f);
10694 %}
10695
10696 ins_pipe(ialu_reg_reg_shift);
10697 %}
10698
10699 // This pattern is automatically generated from aarch64_ad.m4
10700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10701 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10702 iRegIorL2I src1, iRegIorL2I src2,
10703 immI src3) %{
10704 match(Set dst (AndI src1 (LShiftI src2 src3)));
10705
10706 ins_cost(1.9 * INSN_COST);
10707 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10708
10709 ins_encode %{
10710 __ andw(as_Register($dst$$reg),
10711 as_Register($src1$$reg),
10712 as_Register($src2$$reg),
10713 Assembler::LSL,
10714 $src3$$constant & 0x1f);
10715 %}
10716
10717 ins_pipe(ialu_reg_reg_shift);
10718 %}
10719
10720 // This pattern is automatically generated from aarch64_ad.m4
10721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10722 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10723 iRegL src1, iRegL src2,
10724 immI src3) %{
10725 match(Set dst (AndL src1 (LShiftL src2 src3)));
10726
10727 ins_cost(1.9 * INSN_COST);
10728 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10729
10730 ins_encode %{
10731 __ andr(as_Register($dst$$reg),
10732 as_Register($src1$$reg),
10733 as_Register($src2$$reg),
10734 Assembler::LSL,
10735 $src3$$constant & 0x3f);
10736 %}
10737
10738 ins_pipe(ialu_reg_reg_shift);
10739 %}
10740
10741 // This pattern is automatically generated from aarch64_ad.m4
10742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10743 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10744 iRegIorL2I src1, iRegIorL2I src2,
10745 immI src3) %{
10746 match(Set dst (AndI src1 (RotateRight src2 src3)));
10747
10748 ins_cost(1.9 * INSN_COST);
10749 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10750
10751 ins_encode %{
10752 __ andw(as_Register($dst$$reg),
10753 as_Register($src1$$reg),
10754 as_Register($src2$$reg),
10755 Assembler::ROR,
10756 $src3$$constant & 0x1f);
10757 %}
10758
10759 ins_pipe(ialu_reg_reg_shift);
10760 %}
10761
10762 // This pattern is automatically generated from aarch64_ad.m4
10763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10764 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10765 iRegL src1, iRegL src2,
10766 immI src3) %{
10767 match(Set dst (AndL src1 (RotateRight src2 src3)));
10768
10769 ins_cost(1.9 * INSN_COST);
10770 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10771
10772 ins_encode %{
10773 __ andr(as_Register($dst$$reg),
10774 as_Register($src1$$reg),
10775 as_Register($src2$$reg),
10776 Assembler::ROR,
10777 $src3$$constant & 0x3f);
10778 %}
10779
10780 ins_pipe(ialu_reg_reg_shift);
10781 %}
10782
10783 // This pattern is automatically generated from aarch64_ad.m4
10784 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10785 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10786 iRegIorL2I src1, iRegIorL2I src2,
10787 immI src3) %{
10788 match(Set dst (XorI src1 (URShiftI src2 src3)));
10789
10790 ins_cost(1.9 * INSN_COST);
10791 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10792
10793 ins_encode %{
10794 __ eorw(as_Register($dst$$reg),
10795 as_Register($src1$$reg),
10796 as_Register($src2$$reg),
10797 Assembler::LSR,
10798 $src3$$constant & 0x1f);
10799 %}
10800
10801 ins_pipe(ialu_reg_reg_shift);
10802 %}
10803
10804 // This pattern is automatically generated from aarch64_ad.m4
10805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10806 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10807 iRegL src1, iRegL src2,
10808 immI src3) %{
10809 match(Set dst (XorL src1 (URShiftL src2 src3)));
10810
10811 ins_cost(1.9 * INSN_COST);
10812 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10813
10814 ins_encode %{
10815 __ eor(as_Register($dst$$reg),
10816 as_Register($src1$$reg),
10817 as_Register($src2$$reg),
10818 Assembler::LSR,
10819 $src3$$constant & 0x3f);
10820 %}
10821
10822 ins_pipe(ialu_reg_reg_shift);
10823 %}
10824
10825 // This pattern is automatically generated from aarch64_ad.m4
10826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10827 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10828 iRegIorL2I src1, iRegIorL2I src2,
10829 immI src3) %{
10830 match(Set dst (XorI src1 (RShiftI src2 src3)));
10831
10832 ins_cost(1.9 * INSN_COST);
10833 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10834
10835 ins_encode %{
10836 __ eorw(as_Register($dst$$reg),
10837 as_Register($src1$$reg),
10838 as_Register($src2$$reg),
10839 Assembler::ASR,
10840 $src3$$constant & 0x1f);
10841 %}
10842
10843 ins_pipe(ialu_reg_reg_shift);
10844 %}
10845
10846 // This pattern is automatically generated from aarch64_ad.m4
10847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10848 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10849 iRegL src1, iRegL src2,
10850 immI src3) %{
10851 match(Set dst (XorL src1 (RShiftL src2 src3)));
10852
10853 ins_cost(1.9 * INSN_COST);
10854 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10855
10856 ins_encode %{
10857 __ eor(as_Register($dst$$reg),
10858 as_Register($src1$$reg),
10859 as_Register($src2$$reg),
10860 Assembler::ASR,
10861 $src3$$constant & 0x3f);
10862 %}
10863
10864 ins_pipe(ialu_reg_reg_shift);
10865 %}
10866
10867 // This pattern is automatically generated from aarch64_ad.m4
10868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10869 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10870 iRegIorL2I src1, iRegIorL2I src2,
10871 immI src3) %{
10872 match(Set dst (XorI src1 (LShiftI src2 src3)));
10873
10874 ins_cost(1.9 * INSN_COST);
10875 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10876
10877 ins_encode %{
10878 __ eorw(as_Register($dst$$reg),
10879 as_Register($src1$$reg),
10880 as_Register($src2$$reg),
10881 Assembler::LSL,
10882 $src3$$constant & 0x1f);
10883 %}
10884
10885 ins_pipe(ialu_reg_reg_shift);
10886 %}
10887
10888 // This pattern is automatically generated from aarch64_ad.m4
10889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10890 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10891 iRegL src1, iRegL src2,
10892 immI src3) %{
10893 match(Set dst (XorL src1 (LShiftL src2 src3)));
10894
10895 ins_cost(1.9 * INSN_COST);
10896 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10897
10898 ins_encode %{
10899 __ eor(as_Register($dst$$reg),
10900 as_Register($src1$$reg),
10901 as_Register($src2$$reg),
10902 Assembler::LSL,
10903 $src3$$constant & 0x3f);
10904 %}
10905
10906 ins_pipe(ialu_reg_reg_shift);
10907 %}
10908
10909 // This pattern is automatically generated from aarch64_ad.m4
10910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10911 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10912 iRegIorL2I src1, iRegIorL2I src2,
10913 immI src3) %{
10914 match(Set dst (XorI src1 (RotateRight src2 src3)));
10915
10916 ins_cost(1.9 * INSN_COST);
10917 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10918
10919 ins_encode %{
10920 __ eorw(as_Register($dst$$reg),
10921 as_Register($src1$$reg),
10922 as_Register($src2$$reg),
10923 Assembler::ROR,
10924 $src3$$constant & 0x1f);
10925 %}
10926
10927 ins_pipe(ialu_reg_reg_shift);
10928 %}
10929
10930 // This pattern is automatically generated from aarch64_ad.m4
10931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10932 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10933 iRegL src1, iRegL src2,
10934 immI src3) %{
10935 match(Set dst (XorL src1 (RotateRight src2 src3)));
10936
10937 ins_cost(1.9 * INSN_COST);
10938 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10939
10940 ins_encode %{
10941 __ eor(as_Register($dst$$reg),
10942 as_Register($src1$$reg),
10943 as_Register($src2$$reg),
10944 Assembler::ROR,
10945 $src3$$constant & 0x3f);
10946 %}
10947
10948 ins_pipe(ialu_reg_reg_shift);
10949 %}
10950
10951 // This pattern is automatically generated from aarch64_ad.m4
10952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10953 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10954 iRegIorL2I src1, iRegIorL2I src2,
10955 immI src3) %{
10956 match(Set dst (OrI src1 (URShiftI src2 src3)));
10957
10958 ins_cost(1.9 * INSN_COST);
10959 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10960
10961 ins_encode %{
10962 __ orrw(as_Register($dst$$reg),
10963 as_Register($src1$$reg),
10964 as_Register($src2$$reg),
10965 Assembler::LSR,
10966 $src3$$constant & 0x1f);
10967 %}
10968
10969 ins_pipe(ialu_reg_reg_shift);
10970 %}
10971
10972 // This pattern is automatically generated from aarch64_ad.m4
10973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10974 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
10975 iRegL src1, iRegL src2,
10976 immI src3) %{
10977 match(Set dst (OrL src1 (URShiftL src2 src3)));
10978
10979 ins_cost(1.9 * INSN_COST);
10980 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
10981
10982 ins_encode %{
10983 __ orr(as_Register($dst$$reg),
10984 as_Register($src1$$reg),
10985 as_Register($src2$$reg),
10986 Assembler::LSR,
10987 $src3$$constant & 0x3f);
10988 %}
10989
10990 ins_pipe(ialu_reg_reg_shift);
10991 %}
10992
10993 // This pattern is automatically generated from aarch64_ad.m4
10994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10995 instruct OrI_reg_RShift_reg(iRegINoSp dst,
10996 iRegIorL2I src1, iRegIorL2I src2,
10997 immI src3) %{
10998 match(Set dst (OrI src1 (RShiftI src2 src3)));
10999
11000 ins_cost(1.9 * INSN_COST);
11001 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11002
11003 ins_encode %{
11004 __ orrw(as_Register($dst$$reg),
11005 as_Register($src1$$reg),
11006 as_Register($src2$$reg),
11007 Assembler::ASR,
11008 $src3$$constant & 0x1f);
11009 %}
11010
11011 ins_pipe(ialu_reg_reg_shift);
11012 %}
11013
11014 // This pattern is automatically generated from aarch64_ad.m4
11015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11016 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11017 iRegL src1, iRegL src2,
11018 immI src3) %{
11019 match(Set dst (OrL src1 (RShiftL src2 src3)));
11020
11021 ins_cost(1.9 * INSN_COST);
11022 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11023
11024 ins_encode %{
11025 __ orr(as_Register($dst$$reg),
11026 as_Register($src1$$reg),
11027 as_Register($src2$$reg),
11028 Assembler::ASR,
11029 $src3$$constant & 0x3f);
11030 %}
11031
11032 ins_pipe(ialu_reg_reg_shift);
11033 %}
11034
11035 // This pattern is automatically generated from aarch64_ad.m4
11036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11037 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11038 iRegIorL2I src1, iRegIorL2I src2,
11039 immI src3) %{
11040 match(Set dst (OrI src1 (LShiftI src2 src3)));
11041
11042 ins_cost(1.9 * INSN_COST);
11043 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11044
11045 ins_encode %{
11046 __ orrw(as_Register($dst$$reg),
11047 as_Register($src1$$reg),
11048 as_Register($src2$$reg),
11049 Assembler::LSL,
11050 $src3$$constant & 0x1f);
11051 %}
11052
11053 ins_pipe(ialu_reg_reg_shift);
11054 %}
11055
11056 // This pattern is automatically generated from aarch64_ad.m4
11057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11058 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11059 iRegL src1, iRegL src2,
11060 immI src3) %{
11061 match(Set dst (OrL src1 (LShiftL src2 src3)));
11062
11063 ins_cost(1.9 * INSN_COST);
11064 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11065
11066 ins_encode %{
11067 __ orr(as_Register($dst$$reg),
11068 as_Register($src1$$reg),
11069 as_Register($src2$$reg),
11070 Assembler::LSL,
11071 $src3$$constant & 0x3f);
11072 %}
11073
11074 ins_pipe(ialu_reg_reg_shift);
11075 %}
11076
11077 // This pattern is automatically generated from aarch64_ad.m4
11078 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11079 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11080 iRegIorL2I src1, iRegIorL2I src2,
11081 immI src3) %{
11082 match(Set dst (OrI src1 (RotateRight src2 src3)));
11083
11084 ins_cost(1.9 * INSN_COST);
11085 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11086
11087 ins_encode %{
11088 __ orrw(as_Register($dst$$reg),
11089 as_Register($src1$$reg),
11090 as_Register($src2$$reg),
11091 Assembler::ROR,
11092 $src3$$constant & 0x1f);
11093 %}
11094
11095 ins_pipe(ialu_reg_reg_shift);
11096 %}
11097
11098 // This pattern is automatically generated from aarch64_ad.m4
11099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11100 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11101 iRegL src1, iRegL src2,
11102 immI src3) %{
11103 match(Set dst (OrL src1 (RotateRight src2 src3)));
11104
11105 ins_cost(1.9 * INSN_COST);
11106 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11107
11108 ins_encode %{
11109 __ orr(as_Register($dst$$reg),
11110 as_Register($src1$$reg),
11111 as_Register($src2$$reg),
11112 Assembler::ROR,
11113 $src3$$constant & 0x3f);
11114 %}
11115
11116 ins_pipe(ialu_reg_reg_shift);
11117 %}
11118
11119 // This pattern is automatically generated from aarch64_ad.m4
11120 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11121 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11122 iRegIorL2I src1, iRegIorL2I src2,
11123 immI src3) %{
11124 match(Set dst (AddI src1 (URShiftI src2 src3)));
11125
11126 ins_cost(1.9 * INSN_COST);
11127 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11128
11129 ins_encode %{
11130 __ addw(as_Register($dst$$reg),
11131 as_Register($src1$$reg),
11132 as_Register($src2$$reg),
11133 Assembler::LSR,
11134 $src3$$constant & 0x1f);
11135 %}
11136
11137 ins_pipe(ialu_reg_reg_shift);
11138 %}
11139
11140 // This pattern is automatically generated from aarch64_ad.m4
11141 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11142 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11143 iRegL src1, iRegL src2,
11144 immI src3) %{
11145 match(Set dst (AddL src1 (URShiftL src2 src3)));
11146
11147 ins_cost(1.9 * INSN_COST);
11148 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11149
11150 ins_encode %{
11151 __ add(as_Register($dst$$reg),
11152 as_Register($src1$$reg),
11153 as_Register($src2$$reg),
11154 Assembler::LSR,
11155 $src3$$constant & 0x3f);
11156 %}
11157
11158 ins_pipe(ialu_reg_reg_shift);
11159 %}
11160
11161 // This pattern is automatically generated from aarch64_ad.m4
11162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11163 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11164 iRegIorL2I src1, iRegIorL2I src2,
11165 immI src3) %{
11166 match(Set dst (AddI src1 (RShiftI src2 src3)));
11167
11168 ins_cost(1.9 * INSN_COST);
11169 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11170
11171 ins_encode %{
11172 __ addw(as_Register($dst$$reg),
11173 as_Register($src1$$reg),
11174 as_Register($src2$$reg),
11175 Assembler::ASR,
11176 $src3$$constant & 0x1f);
11177 %}
11178
11179 ins_pipe(ialu_reg_reg_shift);
11180 %}
11181
11182 // This pattern is automatically generated from aarch64_ad.m4
11183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11184 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11185 iRegL src1, iRegL src2,
11186 immI src3) %{
11187 match(Set dst (AddL src1 (RShiftL src2 src3)));
11188
11189 ins_cost(1.9 * INSN_COST);
11190 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11191
11192 ins_encode %{
11193 __ add(as_Register($dst$$reg),
11194 as_Register($src1$$reg),
11195 as_Register($src2$$reg),
11196 Assembler::ASR,
11197 $src3$$constant & 0x3f);
11198 %}
11199
11200 ins_pipe(ialu_reg_reg_shift);
11201 %}
11202
11203 // This pattern is automatically generated from aarch64_ad.m4
11204 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11205 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11206 iRegIorL2I src1, iRegIorL2I src2,
11207 immI src3) %{
11208 match(Set dst (AddI src1 (LShiftI src2 src3)));
11209
11210 ins_cost(1.9 * INSN_COST);
11211 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11212
11213 ins_encode %{
11214 __ addw(as_Register($dst$$reg),
11215 as_Register($src1$$reg),
11216 as_Register($src2$$reg),
11217 Assembler::LSL,
11218 $src3$$constant & 0x1f);
11219 %}
11220
11221 ins_pipe(ialu_reg_reg_shift);
11222 %}
11223
11224 // This pattern is automatically generated from aarch64_ad.m4
11225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11226 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11227 iRegL src1, iRegL src2,
11228 immI src3) %{
11229 match(Set dst (AddL src1 (LShiftL src2 src3)));
11230
11231 ins_cost(1.9 * INSN_COST);
11232 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11233
11234 ins_encode %{
11235 __ add(as_Register($dst$$reg),
11236 as_Register($src1$$reg),
11237 as_Register($src2$$reg),
11238 Assembler::LSL,
11239 $src3$$constant & 0x3f);
11240 %}
11241
11242 ins_pipe(ialu_reg_reg_shift);
11243 %}
11244
11245 // This pattern is automatically generated from aarch64_ad.m4
11246 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11247 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11248 iRegIorL2I src1, iRegIorL2I src2,
11249 immI src3) %{
11250 match(Set dst (SubI src1 (URShiftI src2 src3)));
11251
11252 ins_cost(1.9 * INSN_COST);
11253 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11254
11255 ins_encode %{
11256 __ subw(as_Register($dst$$reg),
11257 as_Register($src1$$reg),
11258 as_Register($src2$$reg),
11259 Assembler::LSR,
11260 $src3$$constant & 0x1f);
11261 %}
11262
11263 ins_pipe(ialu_reg_reg_shift);
11264 %}
11265
11266 // This pattern is automatically generated from aarch64_ad.m4
11267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11268 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11269 iRegL src1, iRegL src2,
11270 immI src3) %{
11271 match(Set dst (SubL src1 (URShiftL src2 src3)));
11272
11273 ins_cost(1.9 * INSN_COST);
11274 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11275
11276 ins_encode %{
11277 __ sub(as_Register($dst$$reg),
11278 as_Register($src1$$reg),
11279 as_Register($src2$$reg),
11280 Assembler::LSR,
11281 $src3$$constant & 0x3f);
11282 %}
11283
11284 ins_pipe(ialu_reg_reg_shift);
11285 %}
11286
11287 // This pattern is automatically generated from aarch64_ad.m4
11288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11289 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11290 iRegIorL2I src1, iRegIorL2I src2,
11291 immI src3) %{
11292 match(Set dst (SubI src1 (RShiftI src2 src3)));
11293
11294 ins_cost(1.9 * INSN_COST);
11295 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11296
11297 ins_encode %{
11298 __ subw(as_Register($dst$$reg),
11299 as_Register($src1$$reg),
11300 as_Register($src2$$reg),
11301 Assembler::ASR,
11302 $src3$$constant & 0x1f);
11303 %}
11304
11305 ins_pipe(ialu_reg_reg_shift);
11306 %}
11307
11308 // This pattern is automatically generated from aarch64_ad.m4
11309 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11310 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11311 iRegL src1, iRegL src2,
11312 immI src3) %{
11313 match(Set dst (SubL src1 (RShiftL src2 src3)));
11314
11315 ins_cost(1.9 * INSN_COST);
11316 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11317
11318 ins_encode %{
11319 __ sub(as_Register($dst$$reg),
11320 as_Register($src1$$reg),
11321 as_Register($src2$$reg),
11322 Assembler::ASR,
11323 $src3$$constant & 0x3f);
11324 %}
11325
11326 ins_pipe(ialu_reg_reg_shift);
11327 %}
11328
11329 // This pattern is automatically generated from aarch64_ad.m4
11330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11331 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11332 iRegIorL2I src1, iRegIorL2I src2,
11333 immI src3) %{
11334 match(Set dst (SubI src1 (LShiftI src2 src3)));
11335
11336 ins_cost(1.9 * INSN_COST);
11337 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11338
11339 ins_encode %{
11340 __ subw(as_Register($dst$$reg),
11341 as_Register($src1$$reg),
11342 as_Register($src2$$reg),
11343 Assembler::LSL,
11344 $src3$$constant & 0x1f);
11345 %}
11346
11347 ins_pipe(ialu_reg_reg_shift);
11348 %}
11349
11350 // This pattern is automatically generated from aarch64_ad.m4
11351 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11352 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11353 iRegL src1, iRegL src2,
11354 immI src3) %{
11355 match(Set dst (SubL src1 (LShiftL src2 src3)));
11356
11357 ins_cost(1.9 * INSN_COST);
11358 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11359
11360 ins_encode %{
11361 __ sub(as_Register($dst$$reg),
11362 as_Register($src1$$reg),
11363 as_Register($src2$$reg),
11364 Assembler::LSL,
11365 $src3$$constant & 0x3f);
11366 %}
11367
11368 ins_pipe(ialu_reg_reg_shift);
11369 %}
11370
11371 // This pattern is automatically generated from aarch64_ad.m4
11372 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11373
11374 // Shift Left followed by Shift Right.
11375 // This idiom is used by the compiler for the i2b bytecode etc.
11376 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11377 %{
11378 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11379 ins_cost(INSN_COST * 2);
11380 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11381 ins_encode %{
11382 int lshift = $lshift_count$$constant & 63;
11383 int rshift = $rshift_count$$constant & 63;
11384 int s = 63 - lshift;
11385 int r = (rshift - lshift) & 63;
11386 __ sbfm(as_Register($dst$$reg),
11387 as_Register($src$$reg),
11388 r, s);
11389 %}
11390
11391 ins_pipe(ialu_reg_shift);
11392 %}
11393
11394 // This pattern is automatically generated from aarch64_ad.m4
11395 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11396
11397 // Shift Left followed by Shift Right.
11398 // This idiom is used by the compiler for the i2b bytecode etc.
11399 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11400 %{
11401 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11402 ins_cost(INSN_COST * 2);
11403 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11404 ins_encode %{
11405 int lshift = $lshift_count$$constant & 31;
11406 int rshift = $rshift_count$$constant & 31;
11407 int s = 31 - lshift;
11408 int r = (rshift - lshift) & 31;
11409 __ sbfmw(as_Register($dst$$reg),
11410 as_Register($src$$reg),
11411 r, s);
11412 %}
11413
11414 ins_pipe(ialu_reg_shift);
11415 %}
11416
11417 // This pattern is automatically generated from aarch64_ad.m4
11418 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11419
11420 // Shift Left followed by Shift Right.
11421 // This idiom is used by the compiler for the i2b bytecode etc.
11422 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11423 %{
11424 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11425 ins_cost(INSN_COST * 2);
11426 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11427 ins_encode %{
11428 int lshift = $lshift_count$$constant & 63;
11429 int rshift = $rshift_count$$constant & 63;
11430 int s = 63 - lshift;
11431 int r = (rshift - lshift) & 63;
11432 __ ubfm(as_Register($dst$$reg),
11433 as_Register($src$$reg),
11434 r, s);
11435 %}
11436
11437 ins_pipe(ialu_reg_shift);
11438 %}
11439
11440 // This pattern is automatically generated from aarch64_ad.m4
11441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11442
11443 // Shift Left followed by Shift Right.
11444 // This idiom is used by the compiler for the i2b bytecode etc.
11445 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11446 %{
11447 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11448 ins_cost(INSN_COST * 2);
11449 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11450 ins_encode %{
11451 int lshift = $lshift_count$$constant & 31;
11452 int rshift = $rshift_count$$constant & 31;
11453 int s = 31 - lshift;
11454 int r = (rshift - lshift) & 31;
11455 __ ubfmw(as_Register($dst$$reg),
11456 as_Register($src$$reg),
11457 r, s);
11458 %}
11459
11460 ins_pipe(ialu_reg_shift);
11461 %}
11462
11463 // Bitfield extract with shift & mask
11464
11465 // This pattern is automatically generated from aarch64_ad.m4
11466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11467 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11468 %{
11469 match(Set dst (AndI (URShiftI src rshift) mask));
11470 // Make sure we are not going to exceed what ubfxw can do.
11471 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11472
11473 ins_cost(INSN_COST);
11474 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11475 ins_encode %{
11476 int rshift = $rshift$$constant & 31;
11477 intptr_t mask = $mask$$constant;
11478 int width = exact_log2(mask+1);
11479 __ ubfxw(as_Register($dst$$reg),
11480 as_Register($src$$reg), rshift, width);
11481 %}
11482 ins_pipe(ialu_reg_shift);
11483 %}
11484
11485 // This pattern is automatically generated from aarch64_ad.m4
11486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11487 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11488 %{
11489 match(Set dst (AndL (URShiftL src rshift) mask));
11490 // Make sure we are not going to exceed what ubfx can do.
11491 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11492
11493 ins_cost(INSN_COST);
11494 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11495 ins_encode %{
11496 int rshift = $rshift$$constant & 63;
11497 intptr_t mask = $mask$$constant;
11498 int width = exact_log2_long(mask+1);
11499 __ ubfx(as_Register($dst$$reg),
11500 as_Register($src$$reg), rshift, width);
11501 %}
11502 ins_pipe(ialu_reg_shift);
11503 %}
11504
11505
11506 // This pattern is automatically generated from aarch64_ad.m4
11507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11508
11509 // We can use ubfx when extending an And with a mask when we know mask
11510 // is positive. We know that because immI_bitmask guarantees it.
11511 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11512 %{
11513 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11514 // Make sure we are not going to exceed what ubfxw can do.
11515 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11516
11517 ins_cost(INSN_COST * 2);
11518 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11519 ins_encode %{
11520 int rshift = $rshift$$constant & 31;
11521 intptr_t mask = $mask$$constant;
11522 int width = exact_log2(mask+1);
11523 __ ubfx(as_Register($dst$$reg),
11524 as_Register($src$$reg), rshift, width);
11525 %}
11526 ins_pipe(ialu_reg_shift);
11527 %}
11528
11529
11530 // This pattern is automatically generated from aarch64_ad.m4
11531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11532
11533 // We can use ubfiz when masking by a positive number and then left shifting the result.
11534 // We know that the mask is positive because immI_bitmask guarantees it.
11535 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11536 %{
11537 match(Set dst (LShiftI (AndI src mask) lshift));
11538 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11539
11540 ins_cost(INSN_COST);
11541 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11542 ins_encode %{
11543 int lshift = $lshift$$constant & 31;
11544 intptr_t mask = $mask$$constant;
11545 int width = exact_log2(mask+1);
11546 __ ubfizw(as_Register($dst$$reg),
11547 as_Register($src$$reg), lshift, width);
11548 %}
11549 ins_pipe(ialu_reg_shift);
11550 %}
11551
11552 // This pattern is automatically generated from aarch64_ad.m4
11553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11554
11555 // We can use ubfiz when masking by a positive number and then left shifting the result.
11556 // We know that the mask is positive because immL_bitmask guarantees it.
11557 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11558 %{
11559 match(Set dst (LShiftL (AndL src mask) lshift));
11560 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11561
11562 ins_cost(INSN_COST);
11563 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11564 ins_encode %{
11565 int lshift = $lshift$$constant & 63;
11566 intptr_t mask = $mask$$constant;
11567 int width = exact_log2_long(mask+1);
11568 __ ubfiz(as_Register($dst$$reg),
11569 as_Register($src$$reg), lshift, width);
11570 %}
11571 ins_pipe(ialu_reg_shift);
11572 %}
11573
11574 // This pattern is automatically generated from aarch64_ad.m4
11575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11576
11577 // We can use ubfiz when masking by a positive number and then left shifting the result.
11578 // We know that the mask is positive because immI_bitmask guarantees it.
11579 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11580 %{
11581 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11582 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11583
11584 ins_cost(INSN_COST);
11585 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11586 ins_encode %{
11587 int lshift = $lshift$$constant & 31;
11588 intptr_t mask = $mask$$constant;
11589 int width = exact_log2(mask+1);
11590 __ ubfizw(as_Register($dst$$reg),
11591 as_Register($src$$reg), lshift, width);
11592 %}
11593 ins_pipe(ialu_reg_shift);
11594 %}
11595
11596 // This pattern is automatically generated from aarch64_ad.m4
11597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11598
11599 // We can use ubfiz when masking by a positive number and then left shifting the result.
11600 // We know that the mask is positive because immL_bitmask guarantees it.
11601 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11602 %{
11603 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11604 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11605
11606 ins_cost(INSN_COST);
11607 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11608 ins_encode %{
11609 int lshift = $lshift$$constant & 63;
11610 intptr_t mask = $mask$$constant;
11611 int width = exact_log2_long(mask+1);
11612 __ ubfiz(as_Register($dst$$reg),
11613 as_Register($src$$reg), lshift, width);
11614 %}
11615 ins_pipe(ialu_reg_shift);
11616 %}
11617
11618
11619 // This pattern is automatically generated from aarch64_ad.m4
11620 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11621
11622 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11623 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11624 %{
11625 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11626 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11627
11628 ins_cost(INSN_COST);
11629 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11630 ins_encode %{
11631 int lshift = $lshift$$constant & 63;
11632 intptr_t mask = $mask$$constant;
11633 int width = exact_log2(mask+1);
11634 __ ubfiz(as_Register($dst$$reg),
11635 as_Register($src$$reg), lshift, width);
11636 %}
11637 ins_pipe(ialu_reg_shift);
11638 %}
11639
11640 // This pattern is automatically generated from aarch64_ad.m4
11641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11642
11643 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11644 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11645 %{
11646 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11647 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11648
11649 ins_cost(INSN_COST);
11650 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11651 ins_encode %{
11652 int lshift = $lshift$$constant & 31;
11653 intptr_t mask = $mask$$constant;
11654 int width = exact_log2(mask+1);
11655 __ ubfiz(as_Register($dst$$reg),
11656 as_Register($src$$reg), lshift, width);
11657 %}
11658 ins_pipe(ialu_reg_shift);
11659 %}
11660
11661 // This pattern is automatically generated from aarch64_ad.m4
11662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11663
11664 // Can skip int2long conversions after AND with small bitmask
11665 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11666 %{
11667 match(Set dst (ConvI2L (AndI src msk)));
11668 ins_cost(INSN_COST);
11669 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11670 ins_encode %{
11671 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11672 %}
11673 ins_pipe(ialu_reg_shift);
11674 %}
11675
11676
11677 // Rotations
11678
11679 // This pattern is automatically generated from aarch64_ad.m4
11680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11681 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11682 %{
11683 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11684 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11685
11686 ins_cost(INSN_COST);
11687 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11688
11689 ins_encode %{
11690 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11691 $rshift$$constant & 63);
11692 %}
11693 ins_pipe(ialu_reg_reg_extr);
11694 %}
11695
11696
11697 // This pattern is automatically generated from aarch64_ad.m4
11698 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11699 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11700 %{
11701 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11702 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11703
11704 ins_cost(INSN_COST);
11705 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11706
11707 ins_encode %{
11708 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11709 $rshift$$constant & 31);
11710 %}
11711 ins_pipe(ialu_reg_reg_extr);
11712 %}
11713
11714
11715 // This pattern is automatically generated from aarch64_ad.m4
11716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11717 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11718 %{
11719 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11720 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11721
11722 ins_cost(INSN_COST);
11723 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11724
11725 ins_encode %{
11726 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11727 $rshift$$constant & 63);
11728 %}
11729 ins_pipe(ialu_reg_reg_extr);
11730 %}
11731
11732
11733 // This pattern is automatically generated from aarch64_ad.m4
11734 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11735 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11736 %{
11737 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11738 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11739
11740 ins_cost(INSN_COST);
11741 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11742
11743 ins_encode %{
11744 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11745 $rshift$$constant & 31);
11746 %}
11747 ins_pipe(ialu_reg_reg_extr);
11748 %}
11749
11750 // This pattern is automatically generated from aarch64_ad.m4
11751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11752 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11753 %{
11754 match(Set dst (RotateRight src shift));
11755
11756 ins_cost(INSN_COST);
11757 format %{ "ror $dst, $src, $shift" %}
11758
11759 ins_encode %{
11760 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11761 $shift$$constant & 0x1f);
11762 %}
11763 ins_pipe(ialu_reg_reg_vshift);
11764 %}
11765
11766 // This pattern is automatically generated from aarch64_ad.m4
11767 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11768 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11769 %{
11770 match(Set dst (RotateRight src shift));
11771
11772 ins_cost(INSN_COST);
11773 format %{ "ror $dst, $src, $shift" %}
11774
11775 ins_encode %{
11776 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11777 $shift$$constant & 0x3f);
11778 %}
11779 ins_pipe(ialu_reg_reg_vshift);
11780 %}
11781
11782 // This pattern is automatically generated from aarch64_ad.m4
11783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11784 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11785 %{
11786 match(Set dst (RotateRight src shift));
11787
11788 ins_cost(INSN_COST);
11789 format %{ "ror $dst, $src, $shift" %}
11790
11791 ins_encode %{
11792 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11793 %}
11794 ins_pipe(ialu_reg_reg_vshift);
11795 %}
11796
11797 // This pattern is automatically generated from aarch64_ad.m4
11798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11799 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11800 %{
11801 match(Set dst (RotateRight src shift));
11802
11803 ins_cost(INSN_COST);
11804 format %{ "ror $dst, $src, $shift" %}
11805
11806 ins_encode %{
11807 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11808 %}
11809 ins_pipe(ialu_reg_reg_vshift);
11810 %}
11811
11812 // This pattern is automatically generated from aarch64_ad.m4
11813 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11814 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11815 %{
11816 match(Set dst (RotateLeft src shift));
11817
11818 ins_cost(INSN_COST);
11819 format %{ "rol $dst, $src, $shift" %}
11820
11821 ins_encode %{
11822 __ subw(rscratch1, zr, as_Register($shift$$reg));
11823 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11824 %}
11825 ins_pipe(ialu_reg_reg_vshift);
11826 %}
11827
11828 // This pattern is automatically generated from aarch64_ad.m4
11829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11830 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11831 %{
11832 match(Set dst (RotateLeft src shift));
11833
11834 ins_cost(INSN_COST);
11835 format %{ "rol $dst, $src, $shift" %}
11836
11837 ins_encode %{
11838 __ subw(rscratch1, zr, as_Register($shift$$reg));
11839 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11840 %}
11841 ins_pipe(ialu_reg_reg_vshift);
11842 %}
11843
11844
11845 // Add/subtract (extended)
11846
11847 // This pattern is automatically generated from aarch64_ad.m4
11848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11849 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11850 %{
11851 match(Set dst (AddL src1 (ConvI2L src2)));
11852 ins_cost(INSN_COST);
11853 format %{ "add $dst, $src1, $src2, sxtw" %}
11854
11855 ins_encode %{
11856 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11857 as_Register($src2$$reg), ext::sxtw);
11858 %}
11859 ins_pipe(ialu_reg_reg);
11860 %}
11861
11862 // This pattern is automatically generated from aarch64_ad.m4
11863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11864 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11865 %{
11866 match(Set dst (SubL src1 (ConvI2L src2)));
11867 ins_cost(INSN_COST);
11868 format %{ "sub $dst, $src1, $src2, sxtw" %}
11869
11870 ins_encode %{
11871 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11872 as_Register($src2$$reg), ext::sxtw);
11873 %}
11874 ins_pipe(ialu_reg_reg);
11875 %}
11876
11877 // This pattern is automatically generated from aarch64_ad.m4
11878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11879 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11880 %{
11881 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11882 ins_cost(INSN_COST);
11883 format %{ "add $dst, $src1, $src2, sxth" %}
11884
11885 ins_encode %{
11886 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11887 as_Register($src2$$reg), ext::sxth);
11888 %}
11889 ins_pipe(ialu_reg_reg);
11890 %}
11891
11892 // This pattern is automatically generated from aarch64_ad.m4
11893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11894 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11895 %{
11896 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11897 ins_cost(INSN_COST);
11898 format %{ "add $dst, $src1, $src2, sxtb" %}
11899
11900 ins_encode %{
11901 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11902 as_Register($src2$$reg), ext::sxtb);
11903 %}
11904 ins_pipe(ialu_reg_reg);
11905 %}
11906
11907 // This pattern is automatically generated from aarch64_ad.m4
11908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11909 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11910 %{
11911 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11912 ins_cost(INSN_COST);
11913 format %{ "add $dst, $src1, $src2, uxtb" %}
11914
11915 ins_encode %{
11916 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11917 as_Register($src2$$reg), ext::uxtb);
11918 %}
11919 ins_pipe(ialu_reg_reg);
11920 %}
11921
11922 // This pattern is automatically generated from aarch64_ad.m4
11923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11924 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11925 %{
11926 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11927 ins_cost(INSN_COST);
11928 format %{ "add $dst, $src1, $src2, sxth" %}
11929
11930 ins_encode %{
11931 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11932 as_Register($src2$$reg), ext::sxth);
11933 %}
11934 ins_pipe(ialu_reg_reg);
11935 %}
11936
11937 // This pattern is automatically generated from aarch64_ad.m4
11938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11939 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11940 %{
11941 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11942 ins_cost(INSN_COST);
11943 format %{ "add $dst, $src1, $src2, sxtw" %}
11944
11945 ins_encode %{
11946 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11947 as_Register($src2$$reg), ext::sxtw);
11948 %}
11949 ins_pipe(ialu_reg_reg);
11950 %}
11951
11952 // This pattern is automatically generated from aarch64_ad.m4
11953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11954 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11955 %{
11956 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11957 ins_cost(INSN_COST);
11958 format %{ "add $dst, $src1, $src2, sxtb" %}
11959
11960 ins_encode %{
11961 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11962 as_Register($src2$$reg), ext::sxtb);
11963 %}
11964 ins_pipe(ialu_reg_reg);
11965 %}
11966
11967 // This pattern is automatically generated from aarch64_ad.m4
11968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11969 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11970 %{
11971 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11972 ins_cost(INSN_COST);
11973 format %{ "add $dst, $src1, $src2, uxtb" %}
11974
11975 ins_encode %{
11976 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11977 as_Register($src2$$reg), ext::uxtb);
11978 %}
11979 ins_pipe(ialu_reg_reg);
11980 %}
11981
11982 // This pattern is automatically generated from aarch64_ad.m4
11983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11984 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11985 %{
11986 match(Set dst (AddI src1 (AndI src2 mask)));
11987 ins_cost(INSN_COST);
11988 format %{ "addw $dst, $src1, $src2, uxtb" %}
11989
11990 ins_encode %{
11991 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11992 as_Register($src2$$reg), ext::uxtb);
11993 %}
11994 ins_pipe(ialu_reg_reg);
11995 %}
11996
11997 // This pattern is automatically generated from aarch64_ad.m4
11998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11999 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12000 %{
12001 match(Set dst (AddI src1 (AndI src2 mask)));
12002 ins_cost(INSN_COST);
12003 format %{ "addw $dst, $src1, $src2, uxth" %}
12004
12005 ins_encode %{
12006 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12007 as_Register($src2$$reg), ext::uxth);
12008 %}
12009 ins_pipe(ialu_reg_reg);
12010 %}
12011
12012 // This pattern is automatically generated from aarch64_ad.m4
12013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12014 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12015 %{
12016 match(Set dst (AddL src1 (AndL src2 mask)));
12017 ins_cost(INSN_COST);
12018 format %{ "add $dst, $src1, $src2, uxtb" %}
12019
12020 ins_encode %{
12021 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12022 as_Register($src2$$reg), ext::uxtb);
12023 %}
12024 ins_pipe(ialu_reg_reg);
12025 %}
12026
12027 // This pattern is automatically generated from aarch64_ad.m4
12028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12029 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12030 %{
12031 match(Set dst (AddL src1 (AndL src2 mask)));
12032 ins_cost(INSN_COST);
12033 format %{ "add $dst, $src1, $src2, uxth" %}
12034
12035 ins_encode %{
12036 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12037 as_Register($src2$$reg), ext::uxth);
12038 %}
12039 ins_pipe(ialu_reg_reg);
12040 %}
12041
12042 // This pattern is automatically generated from aarch64_ad.m4
12043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12044 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12045 %{
12046 match(Set dst (AddL src1 (AndL src2 mask)));
12047 ins_cost(INSN_COST);
12048 format %{ "add $dst, $src1, $src2, uxtw" %}
12049
12050 ins_encode %{
12051 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12052 as_Register($src2$$reg), ext::uxtw);
12053 %}
12054 ins_pipe(ialu_reg_reg);
12055 %}
12056
12057 // This pattern is automatically generated from aarch64_ad.m4
12058 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12059 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12060 %{
12061 match(Set dst (SubI src1 (AndI src2 mask)));
12062 ins_cost(INSN_COST);
12063 format %{ "subw $dst, $src1, $src2, uxtb" %}
12064
12065 ins_encode %{
12066 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12067 as_Register($src2$$reg), ext::uxtb);
12068 %}
12069 ins_pipe(ialu_reg_reg);
12070 %}
12071
12072 // This pattern is automatically generated from aarch64_ad.m4
12073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12074 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12075 %{
12076 match(Set dst (SubI src1 (AndI src2 mask)));
12077 ins_cost(INSN_COST);
12078 format %{ "subw $dst, $src1, $src2, uxth" %}
12079
12080 ins_encode %{
12081 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12082 as_Register($src2$$reg), ext::uxth);
12083 %}
12084 ins_pipe(ialu_reg_reg);
12085 %}
12086
12087 // This pattern is automatically generated from aarch64_ad.m4
12088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12089 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12090 %{
12091 match(Set dst (SubL src1 (AndL src2 mask)));
12092 ins_cost(INSN_COST);
12093 format %{ "sub $dst, $src1, $src2, uxtb" %}
12094
12095 ins_encode %{
12096 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12097 as_Register($src2$$reg), ext::uxtb);
12098 %}
12099 ins_pipe(ialu_reg_reg);
12100 %}
12101
12102 // This pattern is automatically generated from aarch64_ad.m4
12103 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12104 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12105 %{
12106 match(Set dst (SubL src1 (AndL src2 mask)));
12107 ins_cost(INSN_COST);
12108 format %{ "sub $dst, $src1, $src2, uxth" %}
12109
12110 ins_encode %{
12111 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12112 as_Register($src2$$reg), ext::uxth);
12113 %}
12114 ins_pipe(ialu_reg_reg);
12115 %}
12116
12117 // This pattern is automatically generated from aarch64_ad.m4
12118 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12119 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12120 %{
12121 match(Set dst (SubL src1 (AndL src2 mask)));
12122 ins_cost(INSN_COST);
12123 format %{ "sub $dst, $src1, $src2, uxtw" %}
12124
12125 ins_encode %{
12126 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12127 as_Register($src2$$reg), ext::uxtw);
12128 %}
12129 ins_pipe(ialu_reg_reg);
12130 %}
12131
12132
12133 // This pattern is automatically generated from aarch64_ad.m4
12134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12135 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12136 %{
12137 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12138 ins_cost(1.9 * INSN_COST);
12139 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12140
12141 ins_encode %{
12142 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12143 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12144 %}
12145 ins_pipe(ialu_reg_reg_shift);
12146 %}
12147
12148 // This pattern is automatically generated from aarch64_ad.m4
12149 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12150 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12151 %{
12152 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12153 ins_cost(1.9 * INSN_COST);
12154 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12155
12156 ins_encode %{
12157 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12158 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12159 %}
12160 ins_pipe(ialu_reg_reg_shift);
12161 %}
12162
12163 // This pattern is automatically generated from aarch64_ad.m4
12164 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12165 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12166 %{
12167 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12168 ins_cost(1.9 * INSN_COST);
12169 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12170
12171 ins_encode %{
12172 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12173 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12174 %}
12175 ins_pipe(ialu_reg_reg_shift);
12176 %}
12177
12178 // This pattern is automatically generated from aarch64_ad.m4
12179 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12180 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12181 %{
12182 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12183 ins_cost(1.9 * INSN_COST);
12184 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12185
12186 ins_encode %{
12187 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12188 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12189 %}
12190 ins_pipe(ialu_reg_reg_shift);
12191 %}
12192
12193 // This pattern is automatically generated from aarch64_ad.m4
12194 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12195 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12196 %{
12197 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12198 ins_cost(1.9 * INSN_COST);
12199 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12200
12201 ins_encode %{
12202 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12203 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12204 %}
12205 ins_pipe(ialu_reg_reg_shift);
12206 %}
12207
12208 // This pattern is automatically generated from aarch64_ad.m4
12209 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12210 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12211 %{
12212 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12213 ins_cost(1.9 * INSN_COST);
12214 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12215
12216 ins_encode %{
12217 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12218 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12219 %}
12220 ins_pipe(ialu_reg_reg_shift);
12221 %}
12222
12223 // This pattern is automatically generated from aarch64_ad.m4
12224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12225 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12226 %{
12227 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12228 ins_cost(1.9 * INSN_COST);
12229 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12230
12231 ins_encode %{
12232 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12233 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12234 %}
12235 ins_pipe(ialu_reg_reg_shift);
12236 %}
12237
12238 // This pattern is automatically generated from aarch64_ad.m4
12239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12240 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12241 %{
12242 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12243 ins_cost(1.9 * INSN_COST);
12244 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12245
12246 ins_encode %{
12247 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12248 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12249 %}
12250 ins_pipe(ialu_reg_reg_shift);
12251 %}
12252
12253 // This pattern is automatically generated from aarch64_ad.m4
12254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12255 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12256 %{
12257 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12258 ins_cost(1.9 * INSN_COST);
12259 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12260
12261 ins_encode %{
12262 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12263 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12264 %}
12265 ins_pipe(ialu_reg_reg_shift);
12266 %}
12267
12268 // This pattern is automatically generated from aarch64_ad.m4
12269 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12270 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12271 %{
12272 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12273 ins_cost(1.9 * INSN_COST);
12274 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12275
12276 ins_encode %{
12277 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12278 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12279 %}
12280 ins_pipe(ialu_reg_reg_shift);
12281 %}
12282
12283 // This pattern is automatically generated from aarch64_ad.m4
12284 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12285 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12286 %{
12287 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12288 ins_cost(1.9 * INSN_COST);
12289 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12290
12291 ins_encode %{
12292 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12293 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12294 %}
12295 ins_pipe(ialu_reg_reg_shift);
12296 %}
12297
12298 // This pattern is automatically generated from aarch64_ad.m4
12299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12300 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12301 %{
12302 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12303 ins_cost(1.9 * INSN_COST);
12304 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12305
12306 ins_encode %{
12307 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12308 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12309 %}
12310 ins_pipe(ialu_reg_reg_shift);
12311 %}
12312
12313 // This pattern is automatically generated from aarch64_ad.m4
12314 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12315 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12316 %{
12317 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12318 ins_cost(1.9 * INSN_COST);
12319 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12320
12321 ins_encode %{
12322 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12323 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12324 %}
12325 ins_pipe(ialu_reg_reg_shift);
12326 %}
12327
12328 // This pattern is automatically generated from aarch64_ad.m4
12329 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12330 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12331 %{
12332 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12333 ins_cost(1.9 * INSN_COST);
12334 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12335
12336 ins_encode %{
12337 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12338 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12339 %}
12340 ins_pipe(ialu_reg_reg_shift);
12341 %}
12342
12343 // This pattern is automatically generated from aarch64_ad.m4
12344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12345 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12346 %{
12347 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12348 ins_cost(1.9 * INSN_COST);
12349 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12350
12351 ins_encode %{
12352 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12353 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12354 %}
12355 ins_pipe(ialu_reg_reg_shift);
12356 %}
12357
12358 // This pattern is automatically generated from aarch64_ad.m4
12359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12360 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12361 %{
12362 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12363 ins_cost(1.9 * INSN_COST);
12364 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12365
12366 ins_encode %{
12367 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12368 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12369 %}
12370 ins_pipe(ialu_reg_reg_shift);
12371 %}
12372
12373 // This pattern is automatically generated from aarch64_ad.m4
12374 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12375 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12376 %{
12377 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12378 ins_cost(1.9 * INSN_COST);
12379 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12380
12381 ins_encode %{
12382 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12383 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12384 %}
12385 ins_pipe(ialu_reg_reg_shift);
12386 %}
12387
12388 // This pattern is automatically generated from aarch64_ad.m4
12389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12390 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12391 %{
12392 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12393 ins_cost(1.9 * INSN_COST);
12394 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12395
12396 ins_encode %{
12397 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12398 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12399 %}
12400 ins_pipe(ialu_reg_reg_shift);
12401 %}
12402
12403 // This pattern is automatically generated from aarch64_ad.m4
12404 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12405 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12406 %{
12407 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12408 ins_cost(1.9 * INSN_COST);
12409 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12410
12411 ins_encode %{
12412 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12413 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12414 %}
12415 ins_pipe(ialu_reg_reg_shift);
12416 %}
12417
12418 // This pattern is automatically generated from aarch64_ad.m4
12419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12420 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12421 %{
12422 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12423 ins_cost(1.9 * INSN_COST);
12424 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12425
12426 ins_encode %{
12427 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12428 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12429 %}
12430 ins_pipe(ialu_reg_reg_shift);
12431 %}
12432
12433 // This pattern is automatically generated from aarch64_ad.m4
12434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12435 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12436 %{
12437 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12438 ins_cost(1.9 * INSN_COST);
12439 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12440
12441 ins_encode %{
12442 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12443 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12444 %}
12445 ins_pipe(ialu_reg_reg_shift);
12446 %}
12447
12448 // This pattern is automatically generated from aarch64_ad.m4
12449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12450 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12451 %{
12452 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12453 ins_cost(1.9 * INSN_COST);
12454 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12455
12456 ins_encode %{
12457 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12458 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12459 %}
12460 ins_pipe(ialu_reg_reg_shift);
12461 %}
12462
12463 // This pattern is automatically generated from aarch64_ad.m4
12464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12465 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12466 %{
12467 effect(DEF dst, USE src1, USE src2, USE cr);
12468 ins_cost(INSN_COST * 2);
12469 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12470
12471 ins_encode %{
12472 __ cselw($dst$$Register,
12473 $src1$$Register,
12474 $src2$$Register,
12475 Assembler::LT);
12476 %}
12477 ins_pipe(icond_reg_reg);
12478 %}
12479
12480 // This pattern is automatically generated from aarch64_ad.m4
12481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12482 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12483 %{
12484 effect(DEF dst, USE src1, USE src2, USE cr);
12485 ins_cost(INSN_COST * 2);
12486 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12487
12488 ins_encode %{
12489 __ cselw($dst$$Register,
12490 $src1$$Register,
12491 $src2$$Register,
12492 Assembler::GT);
12493 %}
12494 ins_pipe(icond_reg_reg);
12495 %}
12496
12497 // This pattern is automatically generated from aarch64_ad.m4
12498 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12499 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12500 %{
12501 effect(DEF dst, USE src1, USE cr);
12502 ins_cost(INSN_COST * 2);
12503 format %{ "cselw $dst, $src1, zr lt\t" %}
12504
12505 ins_encode %{
12506 __ cselw($dst$$Register,
12507 $src1$$Register,
12508 zr,
12509 Assembler::LT);
12510 %}
12511 ins_pipe(icond_reg);
12512 %}
12513
12514 // This pattern is automatically generated from aarch64_ad.m4
12515 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12516 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12517 %{
12518 effect(DEF dst, USE src1, USE cr);
12519 ins_cost(INSN_COST * 2);
12520 format %{ "cselw $dst, $src1, zr gt\t" %}
12521
12522 ins_encode %{
12523 __ cselw($dst$$Register,
12524 $src1$$Register,
12525 zr,
12526 Assembler::GT);
12527 %}
12528 ins_pipe(icond_reg);
12529 %}
12530
12531 // This pattern is automatically generated from aarch64_ad.m4
12532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12533 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12534 %{
12535 effect(DEF dst, USE src1, USE cr);
12536 ins_cost(INSN_COST * 2);
12537 format %{ "csincw $dst, $src1, zr le\t" %}
12538
12539 ins_encode %{
12540 __ csincw($dst$$Register,
12541 $src1$$Register,
12542 zr,
12543 Assembler::LE);
12544 %}
12545 ins_pipe(icond_reg);
12546 %}
12547
12548 // This pattern is automatically generated from aarch64_ad.m4
12549 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12550 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12551 %{
12552 effect(DEF dst, USE src1, USE cr);
12553 ins_cost(INSN_COST * 2);
12554 format %{ "csincw $dst, $src1, zr gt\t" %}
12555
12556 ins_encode %{
12557 __ csincw($dst$$Register,
12558 $src1$$Register,
12559 zr,
12560 Assembler::GT);
12561 %}
12562 ins_pipe(icond_reg);
12563 %}
12564
12565 // This pattern is automatically generated from aarch64_ad.m4
12566 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12567 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12568 %{
12569 effect(DEF dst, USE src1, USE cr);
12570 ins_cost(INSN_COST * 2);
12571 format %{ "csinvw $dst, $src1, zr lt\t" %}
12572
12573 ins_encode %{
12574 __ csinvw($dst$$Register,
12575 $src1$$Register,
12576 zr,
12577 Assembler::LT);
12578 %}
12579 ins_pipe(icond_reg);
12580 %}
12581
12582 // This pattern is automatically generated from aarch64_ad.m4
12583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12584 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12585 %{
12586 effect(DEF dst, USE src1, USE cr);
12587 ins_cost(INSN_COST * 2);
12588 format %{ "csinvw $dst, $src1, zr ge\t" %}
12589
12590 ins_encode %{
12591 __ csinvw($dst$$Register,
12592 $src1$$Register,
12593 zr,
12594 Assembler::GE);
12595 %}
12596 ins_pipe(icond_reg);
12597 %}
12598
12599 // This pattern is automatically generated from aarch64_ad.m4
12600 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12601 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12602 %{
12603 match(Set dst (MinI src imm));
12604 ins_cost(INSN_COST * 3);
12605 expand %{
12606 rFlagsReg cr;
12607 compI_reg_imm0(cr, src);
12608 cmovI_reg_imm0_lt(dst, src, cr);
12609 %}
12610 %}
12611
12612 // This pattern is automatically generated from aarch64_ad.m4
12613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12614 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12615 %{
12616 match(Set dst (MinI imm src));
12617 ins_cost(INSN_COST * 3);
12618 expand %{
12619 rFlagsReg cr;
12620 compI_reg_imm0(cr, src);
12621 cmovI_reg_imm0_lt(dst, src, cr);
12622 %}
12623 %}
12624
12625 // This pattern is automatically generated from aarch64_ad.m4
12626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12627 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12628 %{
12629 match(Set dst (MinI src imm));
12630 ins_cost(INSN_COST * 3);
12631 expand %{
12632 rFlagsReg cr;
12633 compI_reg_imm0(cr, src);
12634 cmovI_reg_imm1_le(dst, src, cr);
12635 %}
12636 %}
12637
12638 // This pattern is automatically generated from aarch64_ad.m4
12639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12640 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12641 %{
12642 match(Set dst (MinI imm src));
12643 ins_cost(INSN_COST * 3);
12644 expand %{
12645 rFlagsReg cr;
12646 compI_reg_imm0(cr, src);
12647 cmovI_reg_imm1_le(dst, src, cr);
12648 %}
12649 %}
12650
12651 // This pattern is automatically generated from aarch64_ad.m4
12652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12653 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12654 %{
12655 match(Set dst (MinI src imm));
12656 ins_cost(INSN_COST * 3);
12657 expand %{
12658 rFlagsReg cr;
12659 compI_reg_imm0(cr, src);
12660 cmovI_reg_immM1_lt(dst, src, cr);
12661 %}
12662 %}
12663
12664 // This pattern is automatically generated from aarch64_ad.m4
12665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12666 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12667 %{
12668 match(Set dst (MinI imm src));
12669 ins_cost(INSN_COST * 3);
12670 expand %{
12671 rFlagsReg cr;
12672 compI_reg_imm0(cr, src);
12673 cmovI_reg_immM1_lt(dst, src, cr);
12674 %}
12675 %}
12676
12677 // This pattern is automatically generated from aarch64_ad.m4
12678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12679 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12680 %{
12681 match(Set dst (MaxI src imm));
12682 ins_cost(INSN_COST * 3);
12683 expand %{
12684 rFlagsReg cr;
12685 compI_reg_imm0(cr, src);
12686 cmovI_reg_imm0_gt(dst, src, cr);
12687 %}
12688 %}
12689
12690 // This pattern is automatically generated from aarch64_ad.m4
12691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12692 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12693 %{
12694 match(Set dst (MaxI imm src));
12695 ins_cost(INSN_COST * 3);
12696 expand %{
12697 rFlagsReg cr;
12698 compI_reg_imm0(cr, src);
12699 cmovI_reg_imm0_gt(dst, src, cr);
12700 %}
12701 %}
12702
12703 // This pattern is automatically generated from aarch64_ad.m4
12704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12705 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12706 %{
12707 match(Set dst (MaxI src imm));
12708 ins_cost(INSN_COST * 3);
12709 expand %{
12710 rFlagsReg cr;
12711 compI_reg_imm0(cr, src);
12712 cmovI_reg_imm1_gt(dst, src, cr);
12713 %}
12714 %}
12715
12716 // This pattern is automatically generated from aarch64_ad.m4
12717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12718 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12719 %{
12720 match(Set dst (MaxI imm src));
12721 ins_cost(INSN_COST * 3);
12722 expand %{
12723 rFlagsReg cr;
12724 compI_reg_imm0(cr, src);
12725 cmovI_reg_imm1_gt(dst, src, cr);
12726 %}
12727 %}
12728
12729 // This pattern is automatically generated from aarch64_ad.m4
12730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12731 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12732 %{
12733 match(Set dst (MaxI src imm));
12734 ins_cost(INSN_COST * 3);
12735 expand %{
12736 rFlagsReg cr;
12737 compI_reg_imm0(cr, src);
12738 cmovI_reg_immM1_ge(dst, src, cr);
12739 %}
12740 %}
12741
12742 // This pattern is automatically generated from aarch64_ad.m4
12743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12744 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12745 %{
12746 match(Set dst (MaxI imm src));
12747 ins_cost(INSN_COST * 3);
12748 expand %{
12749 rFlagsReg cr;
12750 compI_reg_imm0(cr, src);
12751 cmovI_reg_immM1_ge(dst, src, cr);
12752 %}
12753 %}
12754
12755 // This pattern is automatically generated from aarch64_ad.m4
12756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12757 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12758 %{
12759 match(Set dst (ReverseI src));
12760 ins_cost(INSN_COST);
12761 format %{ "rbitw $dst, $src" %}
12762 ins_encode %{
12763 __ rbitw($dst$$Register, $src$$Register);
12764 %}
12765 ins_pipe(ialu_reg);
12766 %}
12767
12768 // This pattern is automatically generated from aarch64_ad.m4
12769 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12770 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12771 %{
12772 match(Set dst (ReverseL src));
12773 ins_cost(INSN_COST);
12774 format %{ "rbit $dst, $src" %}
12775 ins_encode %{
12776 __ rbit($dst$$Register, $src$$Register);
12777 %}
12778 ins_pipe(ialu_reg);
12779 %}
12780
12781
12782 // END This section of the file is automatically generated. Do not edit --------------
12783
12784
12785 // ============================================================================
12786 // Floating Point Arithmetic Instructions
12787
12788 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12789 match(Set dst (AddHF src1 src2));
12790 format %{ "faddh $dst, $src1, $src2" %}
12791 ins_encode %{
12792 __ faddh($dst$$FloatRegister,
12793 $src1$$FloatRegister,
12794 $src2$$FloatRegister);
12795 %}
12796 ins_pipe(fp_dop_reg_reg_s);
12797 %}
12798
12799 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12800 match(Set dst (AddF src1 src2));
12801
12802 ins_cost(INSN_COST * 5);
12803 format %{ "fadds $dst, $src1, $src2" %}
12804
12805 ins_encode %{
12806 __ fadds(as_FloatRegister($dst$$reg),
12807 as_FloatRegister($src1$$reg),
12808 as_FloatRegister($src2$$reg));
12809 %}
12810
12811 ins_pipe(fp_dop_reg_reg_s);
12812 %}
12813
12814 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12815 match(Set dst (AddD src1 src2));
12816
12817 ins_cost(INSN_COST * 5);
12818 format %{ "faddd $dst, $src1, $src2" %}
12819
12820 ins_encode %{
12821 __ faddd(as_FloatRegister($dst$$reg),
12822 as_FloatRegister($src1$$reg),
12823 as_FloatRegister($src2$$reg));
12824 %}
12825
12826 ins_pipe(fp_dop_reg_reg_d);
12827 %}
12828
12829 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12830 match(Set dst (SubHF src1 src2));
12831 format %{ "fsubh $dst, $src1, $src2" %}
12832 ins_encode %{
12833 __ fsubh($dst$$FloatRegister,
12834 $src1$$FloatRegister,
12835 $src2$$FloatRegister);
12836 %}
12837 ins_pipe(fp_dop_reg_reg_s);
12838 %}
12839
12840 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12841 match(Set dst (SubF src1 src2));
12842
12843 ins_cost(INSN_COST * 5);
12844 format %{ "fsubs $dst, $src1, $src2" %}
12845
12846 ins_encode %{
12847 __ fsubs(as_FloatRegister($dst$$reg),
12848 as_FloatRegister($src1$$reg),
12849 as_FloatRegister($src2$$reg));
12850 %}
12851
12852 ins_pipe(fp_dop_reg_reg_s);
12853 %}
12854
12855 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12856 match(Set dst (SubD src1 src2));
12857
12858 ins_cost(INSN_COST * 5);
12859 format %{ "fsubd $dst, $src1, $src2" %}
12860
12861 ins_encode %{
12862 __ fsubd(as_FloatRegister($dst$$reg),
12863 as_FloatRegister($src1$$reg),
12864 as_FloatRegister($src2$$reg));
12865 %}
12866
12867 ins_pipe(fp_dop_reg_reg_d);
12868 %}
12869
12870 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12871 match(Set dst (MulHF src1 src2));
12872 format %{ "fmulh $dst, $src1, $src2" %}
12873 ins_encode %{
12874 __ fmulh($dst$$FloatRegister,
12875 $src1$$FloatRegister,
12876 $src2$$FloatRegister);
12877 %}
12878 ins_pipe(fp_dop_reg_reg_s);
12879 %}
12880
12881 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12882 match(Set dst (MulF src1 src2));
12883
12884 ins_cost(INSN_COST * 6);
12885 format %{ "fmuls $dst, $src1, $src2" %}
12886
12887 ins_encode %{
12888 __ fmuls(as_FloatRegister($dst$$reg),
12889 as_FloatRegister($src1$$reg),
12890 as_FloatRegister($src2$$reg));
12891 %}
12892
12893 ins_pipe(fp_dop_reg_reg_s);
12894 %}
12895
12896 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12897 match(Set dst (MulD src1 src2));
12898
12899 ins_cost(INSN_COST * 6);
12900 format %{ "fmuld $dst, $src1, $src2" %}
12901
12902 ins_encode %{
12903 __ fmuld(as_FloatRegister($dst$$reg),
12904 as_FloatRegister($src1$$reg),
12905 as_FloatRegister($src2$$reg));
12906 %}
12907
12908 ins_pipe(fp_dop_reg_reg_d);
12909 %}
12910
12911 // src1 * src2 + src3 (half-precision float)
12912 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12913 match(Set dst (FmaHF src3 (Binary src1 src2)));
12914 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12915 ins_encode %{
12916 assert(UseFMA, "Needs FMA instructions support.");
12917 __ fmaddh($dst$$FloatRegister,
12918 $src1$$FloatRegister,
12919 $src2$$FloatRegister,
12920 $src3$$FloatRegister);
12921 %}
12922 ins_pipe(pipe_class_default);
12923 %}
12924
12925 // src1 * src2 + src3
12926 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12927 match(Set dst (FmaF src3 (Binary src1 src2)));
12928
12929 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12930
12931 ins_encode %{
12932 assert(UseFMA, "Needs FMA instructions support.");
12933 __ fmadds(as_FloatRegister($dst$$reg),
12934 as_FloatRegister($src1$$reg),
12935 as_FloatRegister($src2$$reg),
12936 as_FloatRegister($src3$$reg));
12937 %}
12938
12939 ins_pipe(pipe_class_default);
12940 %}
12941
12942 // src1 * src2 + src3
12943 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12944 match(Set dst (FmaD src3 (Binary src1 src2)));
12945
12946 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12947
12948 ins_encode %{
12949 assert(UseFMA, "Needs FMA instructions support.");
12950 __ fmaddd(as_FloatRegister($dst$$reg),
12951 as_FloatRegister($src1$$reg),
12952 as_FloatRegister($src2$$reg),
12953 as_FloatRegister($src3$$reg));
12954 %}
12955
12956 ins_pipe(pipe_class_default);
12957 %}
12958
12959 // src1 * (-src2) + src3
12960 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12961 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12962 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12963
12964 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
12965
12966 ins_encode %{
12967 assert(UseFMA, "Needs FMA instructions support.");
12968 __ fmsubs(as_FloatRegister($dst$$reg),
12969 as_FloatRegister($src1$$reg),
12970 as_FloatRegister($src2$$reg),
12971 as_FloatRegister($src3$$reg));
12972 %}
12973
12974 ins_pipe(pipe_class_default);
12975 %}
12976
12977 // src1 * (-src2) + src3
12978 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12979 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12980 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12981
12982 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
12983
12984 ins_encode %{
12985 assert(UseFMA, "Needs FMA instructions support.");
12986 __ fmsubd(as_FloatRegister($dst$$reg),
12987 as_FloatRegister($src1$$reg),
12988 as_FloatRegister($src2$$reg),
12989 as_FloatRegister($src3$$reg));
12990 %}
12991
12992 ins_pipe(pipe_class_default);
12993 %}
12994
12995 // src1 * (-src2) - src3
12996 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
12997 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12998 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
12999
13000 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13001
13002 ins_encode %{
13003 assert(UseFMA, "Needs FMA instructions support.");
13004 __ fnmadds(as_FloatRegister($dst$$reg),
13005 as_FloatRegister($src1$$reg),
13006 as_FloatRegister($src2$$reg),
13007 as_FloatRegister($src3$$reg));
13008 %}
13009
13010 ins_pipe(pipe_class_default);
13011 %}
13012
13013 // src1 * (-src2) - src3
13014 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13015 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13016 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13017
13018 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13019
13020 ins_encode %{
13021 assert(UseFMA, "Needs FMA instructions support.");
13022 __ fnmaddd(as_FloatRegister($dst$$reg),
13023 as_FloatRegister($src1$$reg),
13024 as_FloatRegister($src2$$reg),
13025 as_FloatRegister($src3$$reg));
13026 %}
13027
13028 ins_pipe(pipe_class_default);
13029 %}
13030
13031 // src1 * src2 - src3
13032 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13033 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13034
13035 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13036
13037 ins_encode %{
13038 assert(UseFMA, "Needs FMA instructions support.");
13039 __ fnmsubs(as_FloatRegister($dst$$reg),
13040 as_FloatRegister($src1$$reg),
13041 as_FloatRegister($src2$$reg),
13042 as_FloatRegister($src3$$reg));
13043 %}
13044
13045 ins_pipe(pipe_class_default);
13046 %}
13047
13048 // src1 * src2 - src3
13049 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13050 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13051
13052 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13053
13054 ins_encode %{
13055 assert(UseFMA, "Needs FMA instructions support.");
13056 // n.b. insn name should be fnmsubd
13057 __ fnmsub(as_FloatRegister($dst$$reg),
13058 as_FloatRegister($src1$$reg),
13059 as_FloatRegister($src2$$reg),
13060 as_FloatRegister($src3$$reg));
13061 %}
13062
13063 ins_pipe(pipe_class_default);
13064 %}
13065
13066 // Math.max(HH)H (half-precision float)
13067 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13068 match(Set dst (MaxHF src1 src2));
13069 format %{ "fmaxh $dst, $src1, $src2" %}
13070 ins_encode %{
13071 __ fmaxh($dst$$FloatRegister,
13072 $src1$$FloatRegister,
13073 $src2$$FloatRegister);
13074 %}
13075 ins_pipe(fp_dop_reg_reg_s);
13076 %}
13077
13078 // Math.min(HH)H (half-precision float)
13079 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13080 match(Set dst (MinHF src1 src2));
13081 format %{ "fminh $dst, $src1, $src2" %}
13082 ins_encode %{
13083 __ fminh($dst$$FloatRegister,
13084 $src1$$FloatRegister,
13085 $src2$$FloatRegister);
13086 %}
13087 ins_pipe(fp_dop_reg_reg_s);
13088 %}
13089
13090 // Math.max(FF)F
13091 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13092 match(Set dst (MaxF src1 src2));
13093
13094 format %{ "fmaxs $dst, $src1, $src2" %}
13095 ins_encode %{
13096 __ fmaxs(as_FloatRegister($dst$$reg),
13097 as_FloatRegister($src1$$reg),
13098 as_FloatRegister($src2$$reg));
13099 %}
13100
13101 ins_pipe(fp_dop_reg_reg_s);
13102 %}
13103
13104 // Math.min(FF)F
13105 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13106 match(Set dst (MinF src1 src2));
13107
13108 format %{ "fmins $dst, $src1, $src2" %}
13109 ins_encode %{
13110 __ fmins(as_FloatRegister($dst$$reg),
13111 as_FloatRegister($src1$$reg),
13112 as_FloatRegister($src2$$reg));
13113 %}
13114
13115 ins_pipe(fp_dop_reg_reg_s);
13116 %}
13117
13118 // Math.max(DD)D
13119 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13120 match(Set dst (MaxD src1 src2));
13121
13122 format %{ "fmaxd $dst, $src1, $src2" %}
13123 ins_encode %{
13124 __ fmaxd(as_FloatRegister($dst$$reg),
13125 as_FloatRegister($src1$$reg),
13126 as_FloatRegister($src2$$reg));
13127 %}
13128
13129 ins_pipe(fp_dop_reg_reg_d);
13130 %}
13131
13132 // Math.min(DD)D
13133 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13134 match(Set dst (MinD src1 src2));
13135
13136 format %{ "fmind $dst, $src1, $src2" %}
13137 ins_encode %{
13138 __ fmind(as_FloatRegister($dst$$reg),
13139 as_FloatRegister($src1$$reg),
13140 as_FloatRegister($src2$$reg));
13141 %}
13142
13143 ins_pipe(fp_dop_reg_reg_d);
13144 %}
13145
13146 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13147 match(Set dst (DivHF src1 src2));
13148 format %{ "fdivh $dst, $src1, $src2" %}
13149 ins_encode %{
13150 __ fdivh($dst$$FloatRegister,
13151 $src1$$FloatRegister,
13152 $src2$$FloatRegister);
13153 %}
13154 ins_pipe(fp_div_s);
13155 %}
13156
13157 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13158 match(Set dst (DivF src1 src2));
13159
13160 ins_cost(INSN_COST * 18);
13161 format %{ "fdivs $dst, $src1, $src2" %}
13162
13163 ins_encode %{
13164 __ fdivs(as_FloatRegister($dst$$reg),
13165 as_FloatRegister($src1$$reg),
13166 as_FloatRegister($src2$$reg));
13167 %}
13168
13169 ins_pipe(fp_div_s);
13170 %}
13171
13172 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13173 match(Set dst (DivD src1 src2));
13174
13175 ins_cost(INSN_COST * 32);
13176 format %{ "fdivd $dst, $src1, $src2" %}
13177
13178 ins_encode %{
13179 __ fdivd(as_FloatRegister($dst$$reg),
13180 as_FloatRegister($src1$$reg),
13181 as_FloatRegister($src2$$reg));
13182 %}
13183
13184 ins_pipe(fp_div_d);
13185 %}
13186
13187 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13188 match(Set dst (NegF src));
13189
13190 ins_cost(INSN_COST * 3);
13191 format %{ "fneg $dst, $src" %}
13192
13193 ins_encode %{
13194 __ fnegs(as_FloatRegister($dst$$reg),
13195 as_FloatRegister($src$$reg));
13196 %}
13197
13198 ins_pipe(fp_uop_s);
13199 %}
13200
13201 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13202 match(Set dst (NegD src));
13203
13204 ins_cost(INSN_COST * 3);
13205 format %{ "fnegd $dst, $src" %}
13206
13207 ins_encode %{
13208 __ fnegd(as_FloatRegister($dst$$reg),
13209 as_FloatRegister($src$$reg));
13210 %}
13211
13212 ins_pipe(fp_uop_d);
13213 %}
13214
13215 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13216 %{
13217 match(Set dst (AbsI src));
13218
13219 effect(KILL cr);
13220 ins_cost(INSN_COST * 2);
13221 format %{ "cmpw $src, zr\n\t"
13222 "cnegw $dst, $src, Assembler::LT\t# int abs"
13223 %}
13224
13225 ins_encode %{
13226 __ cmpw(as_Register($src$$reg), zr);
13227 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13228 %}
13229 ins_pipe(pipe_class_default);
13230 %}
13231
13232 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13233 %{
13234 match(Set dst (AbsL src));
13235
13236 effect(KILL cr);
13237 ins_cost(INSN_COST * 2);
13238 format %{ "cmp $src, zr\n\t"
13239 "cneg $dst, $src, Assembler::LT\t# long abs"
13240 %}
13241
13242 ins_encode %{
13243 __ cmp(as_Register($src$$reg), zr);
13244 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13245 %}
13246 ins_pipe(pipe_class_default);
13247 %}
13248
13249 instruct absF_reg(vRegF dst, vRegF src) %{
13250 match(Set dst (AbsF src));
13251
13252 ins_cost(INSN_COST * 3);
13253 format %{ "fabss $dst, $src" %}
13254 ins_encode %{
13255 __ fabss(as_FloatRegister($dst$$reg),
13256 as_FloatRegister($src$$reg));
13257 %}
13258
13259 ins_pipe(fp_uop_s);
13260 %}
13261
13262 instruct absD_reg(vRegD dst, vRegD src) %{
13263 match(Set dst (AbsD src));
13264
13265 ins_cost(INSN_COST * 3);
13266 format %{ "fabsd $dst, $src" %}
13267 ins_encode %{
13268 __ fabsd(as_FloatRegister($dst$$reg),
13269 as_FloatRegister($src$$reg));
13270 %}
13271
13272 ins_pipe(fp_uop_d);
13273 %}
13274
13275 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13276 match(Set dst (AbsF (SubF src1 src2)));
13277
13278 ins_cost(INSN_COST * 3);
13279 format %{ "fabds $dst, $src1, $src2" %}
13280 ins_encode %{
13281 __ fabds(as_FloatRegister($dst$$reg),
13282 as_FloatRegister($src1$$reg),
13283 as_FloatRegister($src2$$reg));
13284 %}
13285
13286 ins_pipe(fp_uop_s);
13287 %}
13288
13289 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13290 match(Set dst (AbsD (SubD src1 src2)));
13291
13292 ins_cost(INSN_COST * 3);
13293 format %{ "fabdd $dst, $src1, $src2" %}
13294 ins_encode %{
13295 __ fabdd(as_FloatRegister($dst$$reg),
13296 as_FloatRegister($src1$$reg),
13297 as_FloatRegister($src2$$reg));
13298 %}
13299
13300 ins_pipe(fp_uop_d);
13301 %}
13302
13303 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13304 match(Set dst (SqrtD src));
13305
13306 ins_cost(INSN_COST * 50);
13307 format %{ "fsqrtd $dst, $src" %}
13308 ins_encode %{
13309 __ fsqrtd(as_FloatRegister($dst$$reg),
13310 as_FloatRegister($src$$reg));
13311 %}
13312
13313 ins_pipe(fp_div_s);
13314 %}
13315
13316 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13317 match(Set dst (SqrtF src));
13318
13319 ins_cost(INSN_COST * 50);
13320 format %{ "fsqrts $dst, $src" %}
13321 ins_encode %{
13322 __ fsqrts(as_FloatRegister($dst$$reg),
13323 as_FloatRegister($src$$reg));
13324 %}
13325
13326 ins_pipe(fp_div_d);
13327 %}
13328
13329 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13330 match(Set dst (SqrtHF src));
13331 format %{ "fsqrth $dst, $src" %}
13332 ins_encode %{
13333 __ fsqrth($dst$$FloatRegister,
13334 $src$$FloatRegister);
13335 %}
13336 ins_pipe(fp_div_s);
13337 %}
13338
13339 // Math.rint, floor, ceil
13340 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13341 match(Set dst (RoundDoubleMode src rmode));
13342 format %{ "frint $dst, $src, $rmode" %}
13343 ins_encode %{
13344 switch ($rmode$$constant) {
13345 case RoundDoubleModeNode::rmode_rint:
13346 __ frintnd(as_FloatRegister($dst$$reg),
13347 as_FloatRegister($src$$reg));
13348 break;
13349 case RoundDoubleModeNode::rmode_floor:
13350 __ frintmd(as_FloatRegister($dst$$reg),
13351 as_FloatRegister($src$$reg));
13352 break;
13353 case RoundDoubleModeNode::rmode_ceil:
13354 __ frintpd(as_FloatRegister($dst$$reg),
13355 as_FloatRegister($src$$reg));
13356 break;
13357 }
13358 %}
13359 ins_pipe(fp_uop_d);
13360 %}
13361
13362 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13363 match(Set dst (CopySignD src1 (Binary src2 zero)));
13364 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13365 format %{ "CopySignD $dst $src1 $src2" %}
13366 ins_encode %{
13367 FloatRegister dst = as_FloatRegister($dst$$reg),
13368 src1 = as_FloatRegister($src1$$reg),
13369 src2 = as_FloatRegister($src2$$reg),
13370 zero = as_FloatRegister($zero$$reg);
13371 __ fnegd(dst, zero);
13372 __ bsl(dst, __ T8B, src2, src1);
13373 %}
13374 ins_pipe(fp_uop_d);
13375 %}
13376
13377 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13378 match(Set dst (CopySignF src1 src2));
13379 effect(TEMP_DEF dst, USE src1, USE src2);
13380 format %{ "CopySignF $dst $src1 $src2" %}
13381 ins_encode %{
13382 FloatRegister dst = as_FloatRegister($dst$$reg),
13383 src1 = as_FloatRegister($src1$$reg),
13384 src2 = as_FloatRegister($src2$$reg);
13385 __ movi(dst, __ T2S, 0x80, 24);
13386 __ bsl(dst, __ T8B, src2, src1);
13387 %}
13388 ins_pipe(fp_uop_d);
13389 %}
13390
13391 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13392 match(Set dst (SignumD src (Binary zero one)));
13393 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13394 format %{ "signumD $dst, $src" %}
13395 ins_encode %{
13396 FloatRegister src = as_FloatRegister($src$$reg),
13397 dst = as_FloatRegister($dst$$reg),
13398 zero = as_FloatRegister($zero$$reg),
13399 one = as_FloatRegister($one$$reg);
13400 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13401 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13402 // Bit selection instruction gets bit from "one" for each enabled bit in
13403 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13404 // NaN the whole "src" will be copied because "dst" is zero. For all other
13405 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13406 // from "src", and all other bits are copied from 1.0.
13407 __ bsl(dst, __ T8B, one, src);
13408 %}
13409 ins_pipe(fp_uop_d);
13410 %}
13411
13412 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13413 match(Set dst (SignumF src (Binary zero one)));
13414 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13415 format %{ "signumF $dst, $src" %}
13416 ins_encode %{
13417 FloatRegister src = as_FloatRegister($src$$reg),
13418 dst = as_FloatRegister($dst$$reg),
13419 zero = as_FloatRegister($zero$$reg),
13420 one = as_FloatRegister($one$$reg);
13421 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13422 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13423 // Bit selection instruction gets bit from "one" for each enabled bit in
13424 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13425 // NaN the whole "src" will be copied because "dst" is zero. For all other
13426 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13427 // from "src", and all other bits are copied from 1.0.
13428 __ bsl(dst, __ T8B, one, src);
13429 %}
13430 ins_pipe(fp_uop_d);
13431 %}
13432
13433 instruct onspinwait() %{
13434 match(OnSpinWait);
13435 ins_cost(INSN_COST);
13436
13437 format %{ "onspinwait" %}
13438
13439 ins_encode %{
13440 __ spin_wait();
13441 %}
13442 ins_pipe(pipe_class_empty);
13443 %}
13444
13445 // ============================================================================
13446 // Logical Instructions
13447
13448 // Integer Logical Instructions
13449
13450 // And Instructions
13451
13452
13453 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13454 match(Set dst (AndI src1 src2));
13455
13456 format %{ "andw $dst, $src1, $src2\t# int" %}
13457
13458 ins_cost(INSN_COST);
13459 ins_encode %{
13460 __ andw(as_Register($dst$$reg),
13461 as_Register($src1$$reg),
13462 as_Register($src2$$reg));
13463 %}
13464
13465 ins_pipe(ialu_reg_reg);
13466 %}
13467
13468 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13469 match(Set dst (AndI src1 src2));
13470
13471 format %{ "andsw $dst, $src1, $src2\t# int" %}
13472
13473 ins_cost(INSN_COST);
13474 ins_encode %{
13475 __ andw(as_Register($dst$$reg),
13476 as_Register($src1$$reg),
13477 (uint64_t)($src2$$constant));
13478 %}
13479
13480 ins_pipe(ialu_reg_imm);
13481 %}
13482
13483 // Or Instructions
13484
13485 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13486 match(Set dst (OrI src1 src2));
13487
13488 format %{ "orrw $dst, $src1, $src2\t# int" %}
13489
13490 ins_cost(INSN_COST);
13491 ins_encode %{
13492 __ orrw(as_Register($dst$$reg),
13493 as_Register($src1$$reg),
13494 as_Register($src2$$reg));
13495 %}
13496
13497 ins_pipe(ialu_reg_reg);
13498 %}
13499
13500 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13501 match(Set dst (OrI src1 src2));
13502
13503 format %{ "orrw $dst, $src1, $src2\t# int" %}
13504
13505 ins_cost(INSN_COST);
13506 ins_encode %{
13507 __ orrw(as_Register($dst$$reg),
13508 as_Register($src1$$reg),
13509 (uint64_t)($src2$$constant));
13510 %}
13511
13512 ins_pipe(ialu_reg_imm);
13513 %}
13514
13515 // Xor Instructions
13516
13517 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13518 match(Set dst (XorI src1 src2));
13519
13520 format %{ "eorw $dst, $src1, $src2\t# int" %}
13521
13522 ins_cost(INSN_COST);
13523 ins_encode %{
13524 __ eorw(as_Register($dst$$reg),
13525 as_Register($src1$$reg),
13526 as_Register($src2$$reg));
13527 %}
13528
13529 ins_pipe(ialu_reg_reg);
13530 %}
13531
13532 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13533 match(Set dst (XorI src1 src2));
13534
13535 format %{ "eorw $dst, $src1, $src2\t# int" %}
13536
13537 ins_cost(INSN_COST);
13538 ins_encode %{
13539 __ eorw(as_Register($dst$$reg),
13540 as_Register($src1$$reg),
13541 (uint64_t)($src2$$constant));
13542 %}
13543
13544 ins_pipe(ialu_reg_imm);
13545 %}
13546
13547 // Long Logical Instructions
13548 // TODO
13549
13550 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13551 match(Set dst (AndL src1 src2));
13552
13553 format %{ "and $dst, $src1, $src2\t# int" %}
13554
13555 ins_cost(INSN_COST);
13556 ins_encode %{
13557 __ andr(as_Register($dst$$reg),
13558 as_Register($src1$$reg),
13559 as_Register($src2$$reg));
13560 %}
13561
13562 ins_pipe(ialu_reg_reg);
13563 %}
13564
13565 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13566 match(Set dst (AndL src1 src2));
13567
13568 format %{ "and $dst, $src1, $src2\t# int" %}
13569
13570 ins_cost(INSN_COST);
13571 ins_encode %{
13572 __ andr(as_Register($dst$$reg),
13573 as_Register($src1$$reg),
13574 (uint64_t)($src2$$constant));
13575 %}
13576
13577 ins_pipe(ialu_reg_imm);
13578 %}
13579
13580 // Or Instructions
13581
13582 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13583 match(Set dst (OrL src1 src2));
13584
13585 format %{ "orr $dst, $src1, $src2\t# int" %}
13586
13587 ins_cost(INSN_COST);
13588 ins_encode %{
13589 __ orr(as_Register($dst$$reg),
13590 as_Register($src1$$reg),
13591 as_Register($src2$$reg));
13592 %}
13593
13594 ins_pipe(ialu_reg_reg);
13595 %}
13596
13597 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13598 match(Set dst (OrL src1 src2));
13599
13600 format %{ "orr $dst, $src1, $src2\t# int" %}
13601
13602 ins_cost(INSN_COST);
13603 ins_encode %{
13604 __ orr(as_Register($dst$$reg),
13605 as_Register($src1$$reg),
13606 (uint64_t)($src2$$constant));
13607 %}
13608
13609 ins_pipe(ialu_reg_imm);
13610 %}
13611
13612 // Xor Instructions
13613
13614 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13615 match(Set dst (XorL src1 src2));
13616
13617 format %{ "eor $dst, $src1, $src2\t# int" %}
13618
13619 ins_cost(INSN_COST);
13620 ins_encode %{
13621 __ eor(as_Register($dst$$reg),
13622 as_Register($src1$$reg),
13623 as_Register($src2$$reg));
13624 %}
13625
13626 ins_pipe(ialu_reg_reg);
13627 %}
13628
13629 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13630 match(Set dst (XorL src1 src2));
13631
13632 ins_cost(INSN_COST);
13633 format %{ "eor $dst, $src1, $src2\t# int" %}
13634
13635 ins_encode %{
13636 __ eor(as_Register($dst$$reg),
13637 as_Register($src1$$reg),
13638 (uint64_t)($src2$$constant));
13639 %}
13640
13641 ins_pipe(ialu_reg_imm);
13642 %}
13643
13644 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13645 %{
13646 match(Set dst (ConvI2L src));
13647
13648 ins_cost(INSN_COST);
13649 format %{ "sxtw $dst, $src\t# i2l" %}
13650 ins_encode %{
13651 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13652 %}
13653 ins_pipe(ialu_reg_shift);
13654 %}
13655
13656 // this pattern occurs in bigmath arithmetic
13657 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13658 %{
13659 match(Set dst (AndL (ConvI2L src) mask));
13660
13661 ins_cost(INSN_COST);
13662 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13663 ins_encode %{
13664 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13665 %}
13666
13667 ins_pipe(ialu_reg_shift);
13668 %}
13669
13670 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13671 match(Set dst (ConvL2I src));
13672
13673 ins_cost(INSN_COST);
13674 format %{ "movw $dst, $src \t// l2i" %}
13675
13676 ins_encode %{
13677 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13678 %}
13679
13680 ins_pipe(ialu_reg);
13681 %}
13682
13683 instruct convD2F_reg(vRegF dst, vRegD src) %{
13684 match(Set dst (ConvD2F src));
13685
13686 ins_cost(INSN_COST * 5);
13687 format %{ "fcvtd $dst, $src \t// d2f" %}
13688
13689 ins_encode %{
13690 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13691 %}
13692
13693 ins_pipe(fp_d2f);
13694 %}
13695
13696 instruct convF2D_reg(vRegD dst, vRegF src) %{
13697 match(Set dst (ConvF2D src));
13698
13699 ins_cost(INSN_COST * 5);
13700 format %{ "fcvts $dst, $src \t// f2d" %}
13701
13702 ins_encode %{
13703 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13704 %}
13705
13706 ins_pipe(fp_f2d);
13707 %}
13708
13709 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13710 match(Set dst (ConvF2I src));
13711
13712 ins_cost(INSN_COST * 5);
13713 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13714
13715 ins_encode %{
13716 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13717 %}
13718
13719 ins_pipe(fp_f2i);
13720 %}
13721
13722 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13723 match(Set dst (ConvF2L src));
13724
13725 ins_cost(INSN_COST * 5);
13726 format %{ "fcvtzs $dst, $src \t// f2l" %}
13727
13728 ins_encode %{
13729 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13730 %}
13731
13732 ins_pipe(fp_f2l);
13733 %}
13734
13735 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13736 match(Set dst (ConvF2HF src));
13737 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13738 "smov $dst, $tmp\t# move result from $tmp to $dst"
13739 %}
13740 effect(TEMP tmp);
13741 ins_encode %{
13742 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13743 %}
13744 ins_pipe(pipe_slow);
13745 %}
13746
13747 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13748 match(Set dst (ConvHF2F src));
13749 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13750 "fcvt $dst, $tmp\t# convert half to single precision"
13751 %}
13752 effect(TEMP tmp);
13753 ins_encode %{
13754 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13755 %}
13756 ins_pipe(pipe_slow);
13757 %}
13758
13759 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13760 match(Set dst (ConvI2F src));
13761
13762 ins_cost(INSN_COST * 5);
13763 format %{ "scvtfws $dst, $src \t// i2f" %}
13764
13765 ins_encode %{
13766 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13767 %}
13768
13769 ins_pipe(fp_i2f);
13770 %}
13771
13772 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13773 match(Set dst (ConvL2F src));
13774
13775 ins_cost(INSN_COST * 5);
13776 format %{ "scvtfs $dst, $src \t// l2f" %}
13777
13778 ins_encode %{
13779 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13780 %}
13781
13782 ins_pipe(fp_l2f);
13783 %}
13784
13785 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13786 match(Set dst (ConvD2I src));
13787
13788 ins_cost(INSN_COST * 5);
13789 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13790
13791 ins_encode %{
13792 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13793 %}
13794
13795 ins_pipe(fp_d2i);
13796 %}
13797
13798 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13799 match(Set dst (ConvD2L src));
13800
13801 ins_cost(INSN_COST * 5);
13802 format %{ "fcvtzd $dst, $src \t// d2l" %}
13803
13804 ins_encode %{
13805 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13806 %}
13807
13808 ins_pipe(fp_d2l);
13809 %}
13810
13811 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13812 match(Set dst (ConvI2D src));
13813
13814 ins_cost(INSN_COST * 5);
13815 format %{ "scvtfwd $dst, $src \t// i2d" %}
13816
13817 ins_encode %{
13818 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13819 %}
13820
13821 ins_pipe(fp_i2d);
13822 %}
13823
13824 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13825 match(Set dst (ConvL2D src));
13826
13827 ins_cost(INSN_COST * 5);
13828 format %{ "scvtfd $dst, $src \t// l2d" %}
13829
13830 ins_encode %{
13831 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13832 %}
13833
13834 ins_pipe(fp_l2d);
13835 %}
13836
13837 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13838 %{
13839 match(Set dst (RoundD src));
13840 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13841 format %{ "java_round_double $dst,$src"%}
13842 ins_encode %{
13843 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13844 as_FloatRegister($ftmp$$reg));
13845 %}
13846 ins_pipe(pipe_slow);
13847 %}
13848
13849 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13850 %{
13851 match(Set dst (RoundF src));
13852 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13853 format %{ "java_round_float $dst,$src"%}
13854 ins_encode %{
13855 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13856 as_FloatRegister($ftmp$$reg));
13857 %}
13858 ins_pipe(pipe_slow);
13859 %}
13860
13861 // stack <-> reg and reg <-> reg shuffles with no conversion
13862
13863 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13864
13865 match(Set dst (MoveF2I src));
13866
13867 effect(DEF dst, USE src);
13868
13869 ins_cost(4 * INSN_COST);
13870
13871 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13872
13873 ins_encode %{
13874 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13875 %}
13876
13877 ins_pipe(iload_reg_reg);
13878
13879 %}
13880
13881 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13882
13883 match(Set dst (MoveI2F src));
13884
13885 effect(DEF dst, USE src);
13886
13887 ins_cost(4 * INSN_COST);
13888
13889 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13890
13891 ins_encode %{
13892 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13893 %}
13894
13895 ins_pipe(pipe_class_memory);
13896
13897 %}
13898
13899 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13900
13901 match(Set dst (MoveD2L src));
13902
13903 effect(DEF dst, USE src);
13904
13905 ins_cost(4 * INSN_COST);
13906
13907 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13908
13909 ins_encode %{
13910 __ ldr($dst$$Register, Address(sp, $src$$disp));
13911 %}
13912
13913 ins_pipe(iload_reg_reg);
13914
13915 %}
13916
13917 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13918
13919 match(Set dst (MoveL2D src));
13920
13921 effect(DEF dst, USE src);
13922
13923 ins_cost(4 * INSN_COST);
13924
13925 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13926
13927 ins_encode %{
13928 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13929 %}
13930
13931 ins_pipe(pipe_class_memory);
13932
13933 %}
13934
13935 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13936
13937 match(Set dst (MoveF2I src));
13938
13939 effect(DEF dst, USE src);
13940
13941 ins_cost(INSN_COST);
13942
13943 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13944
13945 ins_encode %{
13946 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13947 %}
13948
13949 ins_pipe(pipe_class_memory);
13950
13951 %}
13952
13953 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13954
13955 match(Set dst (MoveI2F src));
13956
13957 effect(DEF dst, USE src);
13958
13959 ins_cost(INSN_COST);
13960
13961 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13962
13963 ins_encode %{
13964 __ strw($src$$Register, Address(sp, $dst$$disp));
13965 %}
13966
13967 ins_pipe(istore_reg_reg);
13968
13969 %}
13970
13971 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13972
13973 match(Set dst (MoveD2L src));
13974
13975 effect(DEF dst, USE src);
13976
13977 ins_cost(INSN_COST);
13978
13979 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13980
13981 ins_encode %{
13982 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13983 %}
13984
13985 ins_pipe(pipe_class_memory);
13986
13987 %}
13988
13989 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
13990
13991 match(Set dst (MoveL2D src));
13992
13993 effect(DEF dst, USE src);
13994
13995 ins_cost(INSN_COST);
13996
13997 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
13998
13999 ins_encode %{
14000 __ str($src$$Register, Address(sp, $dst$$disp));
14001 %}
14002
14003 ins_pipe(istore_reg_reg);
14004
14005 %}
14006
14007 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14008
14009 match(Set dst (MoveF2I src));
14010
14011 effect(DEF dst, USE src);
14012
14013 ins_cost(INSN_COST);
14014
14015 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14016
14017 ins_encode %{
14018 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14019 %}
14020
14021 ins_pipe(fp_f2i);
14022
14023 %}
14024
14025 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14026
14027 match(Set dst (MoveI2F src));
14028
14029 effect(DEF dst, USE src);
14030
14031 ins_cost(INSN_COST);
14032
14033 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14034
14035 ins_encode %{
14036 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14037 %}
14038
14039 ins_pipe(fp_i2f);
14040
14041 %}
14042
14043 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14044
14045 match(Set dst (MoveD2L src));
14046
14047 effect(DEF dst, USE src);
14048
14049 ins_cost(INSN_COST);
14050
14051 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14052
14053 ins_encode %{
14054 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14055 %}
14056
14057 ins_pipe(fp_d2l);
14058
14059 %}
14060
14061 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14062
14063 match(Set dst (MoveL2D src));
14064
14065 effect(DEF dst, USE src);
14066
14067 ins_cost(INSN_COST);
14068
14069 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14070
14071 ins_encode %{
14072 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14073 %}
14074
14075 ins_pipe(fp_l2d);
14076
14077 %}
14078
14079 // ============================================================================
14080 // clearing of an array
14081
14082 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14083 %{
14084 match(Set dummy (ClearArray cnt base));
14085 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14086
14087 ins_cost(4 * INSN_COST);
14088 format %{ "ClearArray $cnt, $base" %}
14089
14090 ins_encode %{
14091 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14092 if (tpc == nullptr) {
14093 ciEnv::current()->record_failure("CodeCache is full");
14094 return;
14095 }
14096 %}
14097
14098 ins_pipe(pipe_class_memory);
14099 %}
14100
14101 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14102 %{
14103 predicate((uint64_t)n->in(2)->get_long()
14104 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14105 match(Set dummy (ClearArray cnt base));
14106 effect(TEMP temp, USE_KILL base, KILL cr);
14107
14108 ins_cost(4 * INSN_COST);
14109 format %{ "ClearArray $cnt, $base" %}
14110
14111 ins_encode %{
14112 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14113 if (tpc == nullptr) {
14114 ciEnv::current()->record_failure("CodeCache is full");
14115 return;
14116 }
14117 %}
14118
14119 ins_pipe(pipe_class_memory);
14120 %}
14121
14122 // ============================================================================
14123 // Overflow Math Instructions
14124
14125 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14126 %{
14127 match(Set cr (OverflowAddI op1 op2));
14128
14129 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14130 ins_cost(INSN_COST);
14131 ins_encode %{
14132 __ cmnw($op1$$Register, $op2$$Register);
14133 %}
14134
14135 ins_pipe(icmp_reg_reg);
14136 %}
14137
14138 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14139 %{
14140 match(Set cr (OverflowAddI op1 op2));
14141
14142 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14143 ins_cost(INSN_COST);
14144 ins_encode %{
14145 __ cmnw($op1$$Register, $op2$$constant);
14146 %}
14147
14148 ins_pipe(icmp_reg_imm);
14149 %}
14150
14151 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14152 %{
14153 match(Set cr (OverflowAddL op1 op2));
14154
14155 format %{ "cmn $op1, $op2\t# overflow check long" %}
14156 ins_cost(INSN_COST);
14157 ins_encode %{
14158 __ cmn($op1$$Register, $op2$$Register);
14159 %}
14160
14161 ins_pipe(icmp_reg_reg);
14162 %}
14163
14164 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14165 %{
14166 match(Set cr (OverflowAddL op1 op2));
14167
14168 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14169 ins_cost(INSN_COST);
14170 ins_encode %{
14171 __ adds(zr, $op1$$Register, $op2$$constant);
14172 %}
14173
14174 ins_pipe(icmp_reg_imm);
14175 %}
14176
14177 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14178 %{
14179 match(Set cr (OverflowSubI op1 op2));
14180
14181 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14182 ins_cost(INSN_COST);
14183 ins_encode %{
14184 __ cmpw($op1$$Register, $op2$$Register);
14185 %}
14186
14187 ins_pipe(icmp_reg_reg);
14188 %}
14189
14190 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14191 %{
14192 match(Set cr (OverflowSubI op1 op2));
14193
14194 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14195 ins_cost(INSN_COST);
14196 ins_encode %{
14197 __ cmpw($op1$$Register, $op2$$constant);
14198 %}
14199
14200 ins_pipe(icmp_reg_imm);
14201 %}
14202
14203 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14204 %{
14205 match(Set cr (OverflowSubL op1 op2));
14206
14207 format %{ "cmp $op1, $op2\t# overflow check long" %}
14208 ins_cost(INSN_COST);
14209 ins_encode %{
14210 __ cmp($op1$$Register, $op2$$Register);
14211 %}
14212
14213 ins_pipe(icmp_reg_reg);
14214 %}
14215
14216 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14217 %{
14218 match(Set cr (OverflowSubL op1 op2));
14219
14220 format %{ "cmp $op1, $op2\t# overflow check long" %}
14221 ins_cost(INSN_COST);
14222 ins_encode %{
14223 __ subs(zr, $op1$$Register, $op2$$constant);
14224 %}
14225
14226 ins_pipe(icmp_reg_imm);
14227 %}
14228
14229 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14230 %{
14231 match(Set cr (OverflowSubI zero op1));
14232
14233 format %{ "cmpw zr, $op1\t# overflow check int" %}
14234 ins_cost(INSN_COST);
14235 ins_encode %{
14236 __ cmpw(zr, $op1$$Register);
14237 %}
14238
14239 ins_pipe(icmp_reg_imm);
14240 %}
14241
14242 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14243 %{
14244 match(Set cr (OverflowSubL zero op1));
14245
14246 format %{ "cmp zr, $op1\t# overflow check long" %}
14247 ins_cost(INSN_COST);
14248 ins_encode %{
14249 __ cmp(zr, $op1$$Register);
14250 %}
14251
14252 ins_pipe(icmp_reg_imm);
14253 %}
14254
14255 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14256 %{
14257 match(Set cr (OverflowMulI op1 op2));
14258
14259 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14260 "cmp rscratch1, rscratch1, sxtw\n\t"
14261 "movw rscratch1, #0x80000000\n\t"
14262 "cselw rscratch1, rscratch1, zr, NE\n\t"
14263 "cmpw rscratch1, #1" %}
14264 ins_cost(5 * INSN_COST);
14265 ins_encode %{
14266 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14267 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14268 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14269 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14270 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14271 %}
14272
14273 ins_pipe(pipe_slow);
14274 %}
14275
14276 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14277 %{
14278 match(If cmp (OverflowMulI op1 op2));
14279 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14280 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14281 effect(USE labl, KILL cr);
14282
14283 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14284 "cmp rscratch1, rscratch1, sxtw\n\t"
14285 "b$cmp $labl" %}
14286 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14287 ins_encode %{
14288 Label* L = $labl$$label;
14289 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14290 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14291 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14292 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14293 %}
14294
14295 ins_pipe(pipe_serial);
14296 %}
14297
14298 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14299 %{
14300 match(Set cr (OverflowMulL op1 op2));
14301
14302 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14303 "smulh rscratch2, $op1, $op2\n\t"
14304 "cmp rscratch2, rscratch1, ASR #63\n\t"
14305 "movw rscratch1, #0x80000000\n\t"
14306 "cselw rscratch1, rscratch1, zr, NE\n\t"
14307 "cmpw rscratch1, #1" %}
14308 ins_cost(6 * INSN_COST);
14309 ins_encode %{
14310 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14311 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14312 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14313 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14314 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14315 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14316 %}
14317
14318 ins_pipe(pipe_slow);
14319 %}
14320
14321 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14322 %{
14323 match(If cmp (OverflowMulL op1 op2));
14324 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14325 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14326 effect(USE labl, KILL cr);
14327
14328 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14329 "smulh rscratch2, $op1, $op2\n\t"
14330 "cmp rscratch2, rscratch1, ASR #63\n\t"
14331 "b$cmp $labl" %}
14332 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14333 ins_encode %{
14334 Label* L = $labl$$label;
14335 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14336 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14337 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14338 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14339 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14340 %}
14341
14342 ins_pipe(pipe_serial);
14343 %}
14344
14345 // ============================================================================
14346 // Compare Instructions
14347
14348 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14349 %{
14350 match(Set cr (CmpI op1 op2));
14351
14352 effect(DEF cr, USE op1, USE op2);
14353
14354 ins_cost(INSN_COST);
14355 format %{ "cmpw $op1, $op2" %}
14356
14357 ins_encode(aarch64_enc_cmpw(op1, op2));
14358
14359 ins_pipe(icmp_reg_reg);
14360 %}
14361
14362 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14363 %{
14364 match(Set cr (CmpI op1 zero));
14365
14366 effect(DEF cr, USE op1);
14367
14368 ins_cost(INSN_COST);
14369 format %{ "cmpw $op1, 0" %}
14370
14371 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14372
14373 ins_pipe(icmp_reg_imm);
14374 %}
14375
14376 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14377 %{
14378 match(Set cr (CmpI op1 op2));
14379
14380 effect(DEF cr, USE op1);
14381
14382 ins_cost(INSN_COST);
14383 format %{ "cmpw $op1, $op2" %}
14384
14385 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14386
14387 ins_pipe(icmp_reg_imm);
14388 %}
14389
14390 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14391 %{
14392 match(Set cr (CmpI op1 op2));
14393
14394 effect(DEF cr, USE op1);
14395
14396 ins_cost(INSN_COST * 2);
14397 format %{ "cmpw $op1, $op2" %}
14398
14399 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14400
14401 ins_pipe(icmp_reg_imm);
14402 %}
14403
14404 // Unsigned compare Instructions; really, same as signed compare
14405 // except it should only be used to feed an If or a CMovI which takes a
14406 // cmpOpU.
14407
14408 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14409 %{
14410 match(Set cr (CmpU op1 op2));
14411
14412 effect(DEF cr, USE op1, USE op2);
14413
14414 ins_cost(INSN_COST);
14415 format %{ "cmpw $op1, $op2\t# unsigned" %}
14416
14417 ins_encode(aarch64_enc_cmpw(op1, op2));
14418
14419 ins_pipe(icmp_reg_reg);
14420 %}
14421
14422 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14423 %{
14424 match(Set cr (CmpU op1 zero));
14425
14426 effect(DEF cr, USE op1);
14427
14428 ins_cost(INSN_COST);
14429 format %{ "cmpw $op1, #0\t# unsigned" %}
14430
14431 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14432
14433 ins_pipe(icmp_reg_imm);
14434 %}
14435
14436 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14437 %{
14438 match(Set cr (CmpU op1 op2));
14439
14440 effect(DEF cr, USE op1);
14441
14442 ins_cost(INSN_COST);
14443 format %{ "cmpw $op1, $op2\t# unsigned" %}
14444
14445 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14446
14447 ins_pipe(icmp_reg_imm);
14448 %}
14449
14450 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14451 %{
14452 match(Set cr (CmpU op1 op2));
14453
14454 effect(DEF cr, USE op1);
14455
14456 ins_cost(INSN_COST * 2);
14457 format %{ "cmpw $op1, $op2\t# unsigned" %}
14458
14459 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14460
14461 ins_pipe(icmp_reg_imm);
14462 %}
14463
14464 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14465 %{
14466 match(Set cr (CmpL op1 op2));
14467
14468 effect(DEF cr, USE op1, USE op2);
14469
14470 ins_cost(INSN_COST);
14471 format %{ "cmp $op1, $op2" %}
14472
14473 ins_encode(aarch64_enc_cmp(op1, op2));
14474
14475 ins_pipe(icmp_reg_reg);
14476 %}
14477
14478 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14479 %{
14480 match(Set cr (CmpL op1 zero));
14481
14482 effect(DEF cr, USE op1);
14483
14484 ins_cost(INSN_COST);
14485 format %{ "tst $op1" %}
14486
14487 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14488
14489 ins_pipe(icmp_reg_imm);
14490 %}
14491
14492 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14493 %{
14494 match(Set cr (CmpL op1 op2));
14495
14496 effect(DEF cr, USE op1);
14497
14498 ins_cost(INSN_COST);
14499 format %{ "cmp $op1, $op2" %}
14500
14501 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14502
14503 ins_pipe(icmp_reg_imm);
14504 %}
14505
14506 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14507 %{
14508 match(Set cr (CmpL op1 op2));
14509
14510 effect(DEF cr, USE op1);
14511
14512 ins_cost(INSN_COST * 2);
14513 format %{ "cmp $op1, $op2" %}
14514
14515 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14516
14517 ins_pipe(icmp_reg_imm);
14518 %}
14519
14520 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14521 %{
14522 match(Set cr (CmpUL op1 op2));
14523
14524 effect(DEF cr, USE op1, USE op2);
14525
14526 ins_cost(INSN_COST);
14527 format %{ "cmp $op1, $op2" %}
14528
14529 ins_encode(aarch64_enc_cmp(op1, op2));
14530
14531 ins_pipe(icmp_reg_reg);
14532 %}
14533
14534 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14535 %{
14536 match(Set cr (CmpUL op1 zero));
14537
14538 effect(DEF cr, USE op1);
14539
14540 ins_cost(INSN_COST);
14541 format %{ "tst $op1" %}
14542
14543 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14544
14545 ins_pipe(icmp_reg_imm);
14546 %}
14547
14548 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14549 %{
14550 match(Set cr (CmpUL op1 op2));
14551
14552 effect(DEF cr, USE op1);
14553
14554 ins_cost(INSN_COST);
14555 format %{ "cmp $op1, $op2" %}
14556
14557 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14558
14559 ins_pipe(icmp_reg_imm);
14560 %}
14561
14562 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14563 %{
14564 match(Set cr (CmpUL op1 op2));
14565
14566 effect(DEF cr, USE op1);
14567
14568 ins_cost(INSN_COST * 2);
14569 format %{ "cmp $op1, $op2" %}
14570
14571 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14572
14573 ins_pipe(icmp_reg_imm);
14574 %}
14575
14576 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14577 %{
14578 match(Set cr (CmpP op1 op2));
14579
14580 effect(DEF cr, USE op1, USE op2);
14581
14582 ins_cost(INSN_COST);
14583 format %{ "cmp $op1, $op2\t // ptr" %}
14584
14585 ins_encode(aarch64_enc_cmpp(op1, op2));
14586
14587 ins_pipe(icmp_reg_reg);
14588 %}
14589
14590 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14591 %{
14592 match(Set cr (CmpN op1 op2));
14593
14594 effect(DEF cr, USE op1, USE op2);
14595
14596 ins_cost(INSN_COST);
14597 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14598
14599 ins_encode(aarch64_enc_cmpn(op1, op2));
14600
14601 ins_pipe(icmp_reg_reg);
14602 %}
14603
14604 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14605 %{
14606 match(Set cr (CmpP op1 zero));
14607
14608 effect(DEF cr, USE op1, USE zero);
14609
14610 ins_cost(INSN_COST);
14611 format %{ "cmp $op1, 0\t // ptr" %}
14612
14613 ins_encode(aarch64_enc_testp(op1));
14614
14615 ins_pipe(icmp_reg_imm);
14616 %}
14617
14618 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14619 %{
14620 match(Set cr (CmpN op1 zero));
14621
14622 effect(DEF cr, USE op1, USE zero);
14623
14624 ins_cost(INSN_COST);
14625 format %{ "cmp $op1, 0\t // compressed ptr" %}
14626
14627 ins_encode(aarch64_enc_testn(op1));
14628
14629 ins_pipe(icmp_reg_imm);
14630 %}
14631
14632 // FP comparisons
14633 //
14634 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14635 // using normal cmpOp. See declaration of rFlagsReg for details.
14636
14637 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14638 %{
14639 match(Set cr (CmpF src1 src2));
14640
14641 ins_cost(3 * INSN_COST);
14642 format %{ "fcmps $src1, $src2" %}
14643
14644 ins_encode %{
14645 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14646 %}
14647
14648 ins_pipe(pipe_class_compare);
14649 %}
14650
14651 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14652 %{
14653 match(Set cr (CmpF src1 src2));
14654
14655 ins_cost(3 * INSN_COST);
14656 format %{ "fcmps $src1, 0.0" %}
14657
14658 ins_encode %{
14659 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14660 %}
14661
14662 ins_pipe(pipe_class_compare);
14663 %}
14664 // FROM HERE
14665
14666 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14667 %{
14668 match(Set cr (CmpD src1 src2));
14669
14670 ins_cost(3 * INSN_COST);
14671 format %{ "fcmpd $src1, $src2" %}
14672
14673 ins_encode %{
14674 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14675 %}
14676
14677 ins_pipe(pipe_class_compare);
14678 %}
14679
14680 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14681 %{
14682 match(Set cr (CmpD src1 src2));
14683
14684 ins_cost(3 * INSN_COST);
14685 format %{ "fcmpd $src1, 0.0" %}
14686
14687 ins_encode %{
14688 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14689 %}
14690
14691 ins_pipe(pipe_class_compare);
14692 %}
14693
14694 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14695 %{
14696 match(Set dst (CmpF3 src1 src2));
14697 effect(KILL cr);
14698
14699 ins_cost(5 * INSN_COST);
14700 format %{ "fcmps $src1, $src2\n\t"
14701 "csinvw($dst, zr, zr, eq\n\t"
14702 "csnegw($dst, $dst, $dst, lt)"
14703 %}
14704
14705 ins_encode %{
14706 Label done;
14707 FloatRegister s1 = as_FloatRegister($src1$$reg);
14708 FloatRegister s2 = as_FloatRegister($src2$$reg);
14709 Register d = as_Register($dst$$reg);
14710 __ fcmps(s1, s2);
14711 // installs 0 if EQ else -1
14712 __ csinvw(d, zr, zr, Assembler::EQ);
14713 // keeps -1 if less or unordered else installs 1
14714 __ csnegw(d, d, d, Assembler::LT);
14715 __ bind(done);
14716 %}
14717
14718 ins_pipe(pipe_class_default);
14719
14720 %}
14721
14722 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14723 %{
14724 match(Set dst (CmpD3 src1 src2));
14725 effect(KILL cr);
14726
14727 ins_cost(5 * INSN_COST);
14728 format %{ "fcmpd $src1, $src2\n\t"
14729 "csinvw($dst, zr, zr, eq\n\t"
14730 "csnegw($dst, $dst, $dst, lt)"
14731 %}
14732
14733 ins_encode %{
14734 Label done;
14735 FloatRegister s1 = as_FloatRegister($src1$$reg);
14736 FloatRegister s2 = as_FloatRegister($src2$$reg);
14737 Register d = as_Register($dst$$reg);
14738 __ fcmpd(s1, s2);
14739 // installs 0 if EQ else -1
14740 __ csinvw(d, zr, zr, Assembler::EQ);
14741 // keeps -1 if less or unordered else installs 1
14742 __ csnegw(d, d, d, Assembler::LT);
14743 __ bind(done);
14744 %}
14745 ins_pipe(pipe_class_default);
14746
14747 %}
14748
14749 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14750 %{
14751 match(Set dst (CmpF3 src1 zero));
14752 effect(KILL cr);
14753
14754 ins_cost(5 * INSN_COST);
14755 format %{ "fcmps $src1, 0.0\n\t"
14756 "csinvw($dst, zr, zr, eq\n\t"
14757 "csnegw($dst, $dst, $dst, lt)"
14758 %}
14759
14760 ins_encode %{
14761 Label done;
14762 FloatRegister s1 = as_FloatRegister($src1$$reg);
14763 Register d = as_Register($dst$$reg);
14764 __ fcmps(s1, 0.0);
14765 // installs 0 if EQ else -1
14766 __ csinvw(d, zr, zr, Assembler::EQ);
14767 // keeps -1 if less or unordered else installs 1
14768 __ csnegw(d, d, d, Assembler::LT);
14769 __ bind(done);
14770 %}
14771
14772 ins_pipe(pipe_class_default);
14773
14774 %}
14775
14776 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14777 %{
14778 match(Set dst (CmpD3 src1 zero));
14779 effect(KILL cr);
14780
14781 ins_cost(5 * INSN_COST);
14782 format %{ "fcmpd $src1, 0.0\n\t"
14783 "csinvw($dst, zr, zr, eq\n\t"
14784 "csnegw($dst, $dst, $dst, lt)"
14785 %}
14786
14787 ins_encode %{
14788 Label done;
14789 FloatRegister s1 = as_FloatRegister($src1$$reg);
14790 Register d = as_Register($dst$$reg);
14791 __ fcmpd(s1, 0.0);
14792 // installs 0 if EQ else -1
14793 __ csinvw(d, zr, zr, Assembler::EQ);
14794 // keeps -1 if less or unordered else installs 1
14795 __ csnegw(d, d, d, Assembler::LT);
14796 __ bind(done);
14797 %}
14798 ins_pipe(pipe_class_default);
14799
14800 %}
14801
14802 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14803 %{
14804 match(Set dst (CmpLTMask p q));
14805 effect(KILL cr);
14806
14807 ins_cost(3 * INSN_COST);
14808
14809 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14810 "csetw $dst, lt\n\t"
14811 "subw $dst, zr, $dst"
14812 %}
14813
14814 ins_encode %{
14815 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14816 __ csetw(as_Register($dst$$reg), Assembler::LT);
14817 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14818 %}
14819
14820 ins_pipe(ialu_reg_reg);
14821 %}
14822
14823 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14824 %{
14825 match(Set dst (CmpLTMask src zero));
14826 effect(KILL cr);
14827
14828 ins_cost(INSN_COST);
14829
14830 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14831
14832 ins_encode %{
14833 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14834 %}
14835
14836 ins_pipe(ialu_reg_shift);
14837 %}
14838
14839 // ============================================================================
14840 // Max and Min
14841
14842 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14843
14844 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14845 %{
14846 effect(DEF cr, USE src);
14847 ins_cost(INSN_COST);
14848 format %{ "cmpw $src, 0" %}
14849
14850 ins_encode %{
14851 __ cmpw($src$$Register, 0);
14852 %}
14853 ins_pipe(icmp_reg_imm);
14854 %}
14855
14856 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14857 %{
14858 match(Set dst (MinI src1 src2));
14859 ins_cost(INSN_COST * 3);
14860
14861 expand %{
14862 rFlagsReg cr;
14863 compI_reg_reg(cr, src1, src2);
14864 cmovI_reg_reg_lt(dst, src1, src2, cr);
14865 %}
14866 %}
14867
14868 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14869 %{
14870 match(Set dst (MaxI src1 src2));
14871 ins_cost(INSN_COST * 3);
14872
14873 expand %{
14874 rFlagsReg cr;
14875 compI_reg_reg(cr, src1, src2);
14876 cmovI_reg_reg_gt(dst, src1, src2, cr);
14877 %}
14878 %}
14879
14880
14881 // ============================================================================
14882 // Branch Instructions
14883
14884 // Direct Branch.
14885 instruct branch(label lbl)
14886 %{
14887 match(Goto);
14888
14889 effect(USE lbl);
14890
14891 ins_cost(BRANCH_COST);
14892 format %{ "b $lbl" %}
14893
14894 ins_encode(aarch64_enc_b(lbl));
14895
14896 ins_pipe(pipe_branch);
14897 %}
14898
14899 // Conditional Near Branch
14900 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14901 %{
14902 // Same match rule as `branchConFar'.
14903 match(If cmp cr);
14904
14905 effect(USE lbl);
14906
14907 ins_cost(BRANCH_COST);
14908 // If set to 1 this indicates that the current instruction is a
14909 // short variant of a long branch. This avoids using this
14910 // instruction in first-pass matching. It will then only be used in
14911 // the `Shorten_branches' pass.
14912 // ins_short_branch(1);
14913 format %{ "b$cmp $lbl" %}
14914
14915 ins_encode(aarch64_enc_br_con(cmp, lbl));
14916
14917 ins_pipe(pipe_branch_cond);
14918 %}
14919
14920 // Conditional Near Branch Unsigned
14921 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14922 %{
14923 // Same match rule as `branchConFar'.
14924 match(If cmp cr);
14925
14926 effect(USE lbl);
14927
14928 ins_cost(BRANCH_COST);
14929 // If set to 1 this indicates that the current instruction is a
14930 // short variant of a long branch. This avoids using this
14931 // instruction in first-pass matching. It will then only be used in
14932 // the `Shorten_branches' pass.
14933 // ins_short_branch(1);
14934 format %{ "b$cmp $lbl\t# unsigned" %}
14935
14936 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14937
14938 ins_pipe(pipe_branch_cond);
14939 %}
14940
14941 // Make use of CBZ and CBNZ. These instructions, as well as being
14942 // shorter than (cmp; branch), have the additional benefit of not
14943 // killing the flags.
14944
14945 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14946 match(If cmp (CmpI op1 op2));
14947 effect(USE labl);
14948
14949 ins_cost(BRANCH_COST);
14950 format %{ "cbw$cmp $op1, $labl" %}
14951 ins_encode %{
14952 Label* L = $labl$$label;
14953 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14954 if (cond == Assembler::EQ)
14955 __ cbzw($op1$$Register, *L);
14956 else
14957 __ cbnzw($op1$$Register, *L);
14958 %}
14959 ins_pipe(pipe_cmp_branch);
14960 %}
14961
14962 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14963 match(If cmp (CmpL op1 op2));
14964 effect(USE labl);
14965
14966 ins_cost(BRANCH_COST);
14967 format %{ "cb$cmp $op1, $labl" %}
14968 ins_encode %{
14969 Label* L = $labl$$label;
14970 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14971 if (cond == Assembler::EQ)
14972 __ cbz($op1$$Register, *L);
14973 else
14974 __ cbnz($op1$$Register, *L);
14975 %}
14976 ins_pipe(pipe_cmp_branch);
14977 %}
14978
14979 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
14980 match(If cmp (CmpP op1 op2));
14981 effect(USE labl);
14982
14983 ins_cost(BRANCH_COST);
14984 format %{ "cb$cmp $op1, $labl" %}
14985 ins_encode %{
14986 Label* L = $labl$$label;
14987 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14988 if (cond == Assembler::EQ)
14989 __ cbz($op1$$Register, *L);
14990 else
14991 __ cbnz($op1$$Register, *L);
14992 %}
14993 ins_pipe(pipe_cmp_branch);
14994 %}
14995
14996 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
14997 match(If cmp (CmpN op1 op2));
14998 effect(USE labl);
14999
15000 ins_cost(BRANCH_COST);
15001 format %{ "cbw$cmp $op1, $labl" %}
15002 ins_encode %{
15003 Label* L = $labl$$label;
15004 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15005 if (cond == Assembler::EQ)
15006 __ cbzw($op1$$Register, *L);
15007 else
15008 __ cbnzw($op1$$Register, *L);
15009 %}
15010 ins_pipe(pipe_cmp_branch);
15011 %}
15012
15013 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15014 match(If cmp (CmpP (DecodeN oop) zero));
15015 effect(USE labl);
15016
15017 ins_cost(BRANCH_COST);
15018 format %{ "cb$cmp $oop, $labl" %}
15019 ins_encode %{
15020 Label* L = $labl$$label;
15021 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15022 if (cond == Assembler::EQ)
15023 __ cbzw($oop$$Register, *L);
15024 else
15025 __ cbnzw($oop$$Register, *L);
15026 %}
15027 ins_pipe(pipe_cmp_branch);
15028 %}
15029
15030 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15031 match(If cmp (CmpU op1 op2));
15032 effect(USE labl);
15033
15034 ins_cost(BRANCH_COST);
15035 format %{ "cbw$cmp $op1, $labl" %}
15036 ins_encode %{
15037 Label* L = $labl$$label;
15038 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15039 if (cond == Assembler::EQ || cond == Assembler::LS) {
15040 __ cbzw($op1$$Register, *L);
15041 } else {
15042 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15043 __ cbnzw($op1$$Register, *L);
15044 }
15045 %}
15046 ins_pipe(pipe_cmp_branch);
15047 %}
15048
15049 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15050 match(If cmp (CmpUL op1 op2));
15051 effect(USE labl);
15052
15053 ins_cost(BRANCH_COST);
15054 format %{ "cb$cmp $op1, $labl" %}
15055 ins_encode %{
15056 Label* L = $labl$$label;
15057 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15058 if (cond == Assembler::EQ || cond == Assembler::LS) {
15059 __ cbz($op1$$Register, *L);
15060 } else {
15061 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15062 __ cbnz($op1$$Register, *L);
15063 }
15064 %}
15065 ins_pipe(pipe_cmp_branch);
15066 %}
15067
15068 // Test bit and Branch
15069
15070 // Patterns for short (< 32KiB) variants
15071 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15072 match(If cmp (CmpL op1 op2));
15073 effect(USE labl);
15074
15075 ins_cost(BRANCH_COST);
15076 format %{ "cb$cmp $op1, $labl # long" %}
15077 ins_encode %{
15078 Label* L = $labl$$label;
15079 Assembler::Condition cond =
15080 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15081 __ tbr(cond, $op1$$Register, 63, *L);
15082 %}
15083 ins_pipe(pipe_cmp_branch);
15084 ins_short_branch(1);
15085 %}
15086
15087 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15088 match(If cmp (CmpI op1 op2));
15089 effect(USE labl);
15090
15091 ins_cost(BRANCH_COST);
15092 format %{ "cb$cmp $op1, $labl # int" %}
15093 ins_encode %{
15094 Label* L = $labl$$label;
15095 Assembler::Condition cond =
15096 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15097 __ tbr(cond, $op1$$Register, 31, *L);
15098 %}
15099 ins_pipe(pipe_cmp_branch);
15100 ins_short_branch(1);
15101 %}
15102
15103 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15104 match(If cmp (CmpL (AndL op1 op2) op3));
15105 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15106 effect(USE labl);
15107
15108 ins_cost(BRANCH_COST);
15109 format %{ "tb$cmp $op1, $op2, $labl" %}
15110 ins_encode %{
15111 Label* L = $labl$$label;
15112 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15113 int bit = exact_log2_long($op2$$constant);
15114 __ tbr(cond, $op1$$Register, bit, *L);
15115 %}
15116 ins_pipe(pipe_cmp_branch);
15117 ins_short_branch(1);
15118 %}
15119
15120 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15121 match(If cmp (CmpI (AndI op1 op2) op3));
15122 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15123 effect(USE labl);
15124
15125 ins_cost(BRANCH_COST);
15126 format %{ "tb$cmp $op1, $op2, $labl" %}
15127 ins_encode %{
15128 Label* L = $labl$$label;
15129 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15130 int bit = exact_log2((juint)$op2$$constant);
15131 __ tbr(cond, $op1$$Register, bit, *L);
15132 %}
15133 ins_pipe(pipe_cmp_branch);
15134 ins_short_branch(1);
15135 %}
15136
15137 // And far variants
15138 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15139 match(If cmp (CmpL op1 op2));
15140 effect(USE labl);
15141
15142 ins_cost(BRANCH_COST);
15143 format %{ "cb$cmp $op1, $labl # long" %}
15144 ins_encode %{
15145 Label* L = $labl$$label;
15146 Assembler::Condition cond =
15147 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15148 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15149 %}
15150 ins_pipe(pipe_cmp_branch);
15151 %}
15152
15153 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15154 match(If cmp (CmpI op1 op2));
15155 effect(USE labl);
15156
15157 ins_cost(BRANCH_COST);
15158 format %{ "cb$cmp $op1, $labl # int" %}
15159 ins_encode %{
15160 Label* L = $labl$$label;
15161 Assembler::Condition cond =
15162 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15163 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15164 %}
15165 ins_pipe(pipe_cmp_branch);
15166 %}
15167
15168 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15169 match(If cmp (CmpL (AndL op1 op2) op3));
15170 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15171 effect(USE labl);
15172
15173 ins_cost(BRANCH_COST);
15174 format %{ "tb$cmp $op1, $op2, $labl" %}
15175 ins_encode %{
15176 Label* L = $labl$$label;
15177 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15178 int bit = exact_log2_long($op2$$constant);
15179 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15180 %}
15181 ins_pipe(pipe_cmp_branch);
15182 %}
15183
15184 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15185 match(If cmp (CmpI (AndI op1 op2) op3));
15186 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15187 effect(USE labl);
15188
15189 ins_cost(BRANCH_COST);
15190 format %{ "tb$cmp $op1, $op2, $labl" %}
15191 ins_encode %{
15192 Label* L = $labl$$label;
15193 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15194 int bit = exact_log2((juint)$op2$$constant);
15195 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15196 %}
15197 ins_pipe(pipe_cmp_branch);
15198 %}
15199
15200 // Test bits
15201
15202 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15203 match(Set cr (CmpL (AndL op1 op2) op3));
15204 predicate(Assembler::operand_valid_for_logical_immediate
15205 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15206
15207 ins_cost(INSN_COST);
15208 format %{ "tst $op1, $op2 # long" %}
15209 ins_encode %{
15210 __ tst($op1$$Register, $op2$$constant);
15211 %}
15212 ins_pipe(ialu_reg_reg);
15213 %}
15214
15215 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15216 match(Set cr (CmpI (AndI op1 op2) op3));
15217 predicate(Assembler::operand_valid_for_logical_immediate
15218 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15219
15220 ins_cost(INSN_COST);
15221 format %{ "tst $op1, $op2 # int" %}
15222 ins_encode %{
15223 __ tstw($op1$$Register, $op2$$constant);
15224 %}
15225 ins_pipe(ialu_reg_reg);
15226 %}
15227
15228 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15229 match(Set cr (CmpL (AndL op1 op2) op3));
15230
15231 ins_cost(INSN_COST);
15232 format %{ "tst $op1, $op2 # long" %}
15233 ins_encode %{
15234 __ tst($op1$$Register, $op2$$Register);
15235 %}
15236 ins_pipe(ialu_reg_reg);
15237 %}
15238
15239 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15240 match(Set cr (CmpI (AndI op1 op2) op3));
15241
15242 ins_cost(INSN_COST);
15243 format %{ "tstw $op1, $op2 # int" %}
15244 ins_encode %{
15245 __ tstw($op1$$Register, $op2$$Register);
15246 %}
15247 ins_pipe(ialu_reg_reg);
15248 %}
15249
15250
15251 // Conditional Far Branch
15252 // Conditional Far Branch Unsigned
15253 // TODO: fixme
15254
15255 // counted loop end branch near
15256 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15257 %{
15258 match(CountedLoopEnd cmp cr);
15259
15260 effect(USE lbl);
15261
15262 ins_cost(BRANCH_COST);
15263 // short variant.
15264 // ins_short_branch(1);
15265 format %{ "b$cmp $lbl \t// counted loop end" %}
15266
15267 ins_encode(aarch64_enc_br_con(cmp, lbl));
15268
15269 ins_pipe(pipe_branch);
15270 %}
15271
15272 // counted loop end branch far
15273 // TODO: fixme
15274
15275 // ============================================================================
15276 // inlined locking and unlocking
15277
15278 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15279 %{
15280 match(Set cr (FastLock object box));
15281 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15282
15283 ins_cost(5 * INSN_COST);
15284 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15285
15286 ins_encode %{
15287 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15288 %}
15289
15290 ins_pipe(pipe_serial);
15291 %}
15292
15293 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15294 %{
15295 match(Set cr (FastUnlock object box));
15296 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15297
15298 ins_cost(5 * INSN_COST);
15299 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15300
15301 ins_encode %{
15302 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15303 %}
15304
15305 ins_pipe(pipe_serial);
15306 %}
15307
15308 // ============================================================================
15309 // Safepoint Instructions
15310
15311 // TODO
15312 // provide a near and far version of this code
15313
15314 instruct safePoint(rFlagsReg cr, iRegP poll)
15315 %{
15316 match(SafePoint poll);
15317 effect(KILL cr);
15318
15319 format %{
15320 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15321 %}
15322 ins_encode %{
15323 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15324 %}
15325 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15326 %}
15327
15328
15329 // ============================================================================
15330 // Procedure Call/Return Instructions
15331
15332 // Call Java Static Instruction
15333
15334 instruct CallStaticJavaDirect(method meth)
15335 %{
15336 match(CallStaticJava);
15337
15338 effect(USE meth);
15339
15340 ins_cost(CALL_COST);
15341
15342 format %{ "call,static $meth \t// ==> " %}
15343
15344 ins_encode(aarch64_enc_java_static_call(meth),
15345 aarch64_enc_call_epilog);
15346
15347 ins_pipe(pipe_class_call);
15348 %}
15349
15350 // TO HERE
15351
15352 // Call Java Dynamic Instruction
15353 instruct CallDynamicJavaDirect(method meth)
15354 %{
15355 match(CallDynamicJava);
15356
15357 effect(USE meth);
15358
15359 ins_cost(CALL_COST);
15360
15361 format %{ "CALL,dynamic $meth \t// ==> " %}
15362
15363 ins_encode(aarch64_enc_java_dynamic_call(meth),
15364 aarch64_enc_call_epilog);
15365
15366 ins_pipe(pipe_class_call);
15367 %}
15368
15369 // Call Runtime Instruction
15370
15371 instruct CallRuntimeDirect(method meth)
15372 %{
15373 match(CallRuntime);
15374
15375 effect(USE meth);
15376
15377 ins_cost(CALL_COST);
15378
15379 format %{ "CALL, runtime $meth" %}
15380
15381 ins_encode( aarch64_enc_java_to_runtime(meth) );
15382
15383 ins_pipe(pipe_class_call);
15384 %}
15385
15386 // Call Runtime Instruction
15387
15388 instruct CallLeafDirect(method meth)
15389 %{
15390 match(CallLeaf);
15391
15392 effect(USE meth);
15393
15394 ins_cost(CALL_COST);
15395
15396 format %{ "CALL, runtime leaf $meth" %}
15397
15398 ins_encode( aarch64_enc_java_to_runtime(meth) );
15399
15400 ins_pipe(pipe_class_call);
15401 %}
15402
15403 // Call Runtime Instruction without safepoint and with vector arguments
15404 instruct CallLeafDirectVector(method meth)
15405 %{
15406 match(CallLeafVector);
15407
15408 effect(USE meth);
15409
15410 ins_cost(CALL_COST);
15411
15412 format %{ "CALL, runtime leaf vector $meth" %}
15413
15414 ins_encode(aarch64_enc_java_to_runtime(meth));
15415
15416 ins_pipe(pipe_class_call);
15417 %}
15418
15419 // Call Runtime Instruction
15420
15421 instruct CallLeafNoFPDirect(method meth)
15422 %{
15423 match(CallLeafNoFP);
15424
15425 effect(USE meth);
15426
15427 ins_cost(CALL_COST);
15428
15429 format %{ "CALL, runtime leaf nofp $meth" %}
15430
15431 ins_encode( aarch64_enc_java_to_runtime(meth) );
15432
15433 ins_pipe(pipe_class_call);
15434 %}
15435
15436 // Tail Call; Jump from runtime stub to Java code.
15437 // Also known as an 'interprocedural jump'.
15438 // Target of jump will eventually return to caller.
15439 // TailJump below removes the return address.
15440 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15441 // emitted just above the TailCall which has reset rfp to the caller state.
15442 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15443 %{
15444 match(TailCall jump_target method_ptr);
15445
15446 ins_cost(CALL_COST);
15447
15448 format %{ "br $jump_target\t# $method_ptr holds method" %}
15449
15450 ins_encode(aarch64_enc_tail_call(jump_target));
15451
15452 ins_pipe(pipe_class_call);
15453 %}
15454
15455 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15456 %{
15457 match(TailJump jump_target ex_oop);
15458
15459 ins_cost(CALL_COST);
15460
15461 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15462
15463 ins_encode(aarch64_enc_tail_jmp(jump_target));
15464
15465 ins_pipe(pipe_class_call);
15466 %}
15467
15468 // Forward exception.
15469 instruct ForwardExceptionjmp()
15470 %{
15471 match(ForwardException);
15472 ins_cost(CALL_COST);
15473
15474 format %{ "b forward_exception_stub" %}
15475 ins_encode %{
15476 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15477 %}
15478 ins_pipe(pipe_class_call);
15479 %}
15480
15481 // Create exception oop: created by stack-crawling runtime code.
15482 // Created exception is now available to this handler, and is setup
15483 // just prior to jumping to this handler. No code emitted.
15484 // TODO check
15485 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15486 instruct CreateException(iRegP_R0 ex_oop)
15487 %{
15488 match(Set ex_oop (CreateEx));
15489
15490 format %{ " -- \t// exception oop; no code emitted" %}
15491
15492 size(0);
15493
15494 ins_encode( /*empty*/ );
15495
15496 ins_pipe(pipe_class_empty);
15497 %}
15498
15499 // Rethrow exception: The exception oop will come in the first
15500 // argument position. Then JUMP (not call) to the rethrow stub code.
15501 instruct RethrowException() %{
15502 match(Rethrow);
15503 ins_cost(CALL_COST);
15504
15505 format %{ "b rethrow_stub" %}
15506
15507 ins_encode( aarch64_enc_rethrow() );
15508
15509 ins_pipe(pipe_class_call);
15510 %}
15511
15512
15513 // Return Instruction
15514 // epilog node loads ret address into lr as part of frame pop
15515 instruct Ret()
15516 %{
15517 match(Return);
15518
15519 format %{ "ret\t// return register" %}
15520
15521 ins_encode( aarch64_enc_ret() );
15522
15523 ins_pipe(pipe_branch);
15524 %}
15525
15526 // Die now.
15527 instruct ShouldNotReachHere() %{
15528 match(Halt);
15529
15530 ins_cost(CALL_COST);
15531 format %{ "ShouldNotReachHere" %}
15532
15533 ins_encode %{
15534 if (is_reachable()) {
15535 const char* str = __ code_string(_halt_reason);
15536 __ stop(str);
15537 }
15538 %}
15539
15540 ins_pipe(pipe_class_default);
15541 %}
15542
15543 // ============================================================================
15544 // Partial Subtype Check
15545 //
15546 // superklass array for an instance of the superklass. Set a hidden
15547 // internal cache on a hit (cache is checked with exposed code in
15548 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15549 // encoding ALSO sets flags.
15550
15551 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15552 %{
15553 match(Set result (PartialSubtypeCheck sub super));
15554 predicate(!UseSecondarySupersTable);
15555 effect(KILL cr, KILL temp);
15556
15557 ins_cost(20 * INSN_COST); // slightly larger than the next version
15558 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15559
15560 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15561
15562 opcode(0x1); // Force zero of result reg on hit
15563
15564 ins_pipe(pipe_class_memory);
15565 %}
15566
15567 // Two versions of partialSubtypeCheck, both used when we need to
15568 // search for a super class in the secondary supers array. The first
15569 // is used when we don't know _a priori_ the class being searched
15570 // for. The second, far more common, is used when we do know: this is
15571 // used for instanceof, checkcast, and any case where C2 can determine
15572 // it by constant propagation.
15573
15574 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15575 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15576 rFlagsReg cr)
15577 %{
15578 match(Set result (PartialSubtypeCheck sub super));
15579 predicate(UseSecondarySupersTable);
15580 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15581
15582 ins_cost(10 * INSN_COST); // slightly larger than the next version
15583 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15584
15585 ins_encode %{
15586 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15587 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15588 $vtemp$$FloatRegister,
15589 $result$$Register, /*L_success*/nullptr);
15590 %}
15591
15592 ins_pipe(pipe_class_memory);
15593 %}
15594
15595 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15596 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15597 rFlagsReg cr)
15598 %{
15599 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15600 predicate(UseSecondarySupersTable);
15601 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15602
15603 ins_cost(5 * INSN_COST); // smaller than the next version
15604 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15605
15606 ins_encode %{
15607 bool success = false;
15608 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15609 if (InlineSecondarySupersTest) {
15610 success =
15611 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15612 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15613 $vtemp$$FloatRegister,
15614 $result$$Register,
15615 super_klass_slot);
15616 } else {
15617 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15618 success = (call != nullptr);
15619 }
15620 if (!success) {
15621 ciEnv::current()->record_failure("CodeCache is full");
15622 return;
15623 }
15624 %}
15625
15626 ins_pipe(pipe_class_memory);
15627 %}
15628
15629 // Intrisics for String.compareTo()
15630
15631 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15632 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15633 %{
15634 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15635 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15636 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15637
15638 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15639 ins_encode %{
15640 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15641 __ string_compare($str1$$Register, $str2$$Register,
15642 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15643 $tmp1$$Register, $tmp2$$Register,
15644 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15645 %}
15646 ins_pipe(pipe_class_memory);
15647 %}
15648
15649 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15650 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15651 %{
15652 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15653 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15654 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15655
15656 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15657 ins_encode %{
15658 __ string_compare($str1$$Register, $str2$$Register,
15659 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15660 $tmp1$$Register, $tmp2$$Register,
15661 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15662 %}
15663 ins_pipe(pipe_class_memory);
15664 %}
15665
15666 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15667 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15668 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15669 %{
15670 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15671 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15672 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15673 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15674
15675 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15676 ins_encode %{
15677 __ string_compare($str1$$Register, $str2$$Register,
15678 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15679 $tmp1$$Register, $tmp2$$Register,
15680 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15681 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15682 %}
15683 ins_pipe(pipe_class_memory);
15684 %}
15685
15686 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15687 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15688 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15689 %{
15690 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15691 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15692 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15693 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15694
15695 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15696 ins_encode %{
15697 __ string_compare($str1$$Register, $str2$$Register,
15698 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15699 $tmp1$$Register, $tmp2$$Register,
15700 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15701 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15702 %}
15703 ins_pipe(pipe_class_memory);
15704 %}
15705
15706 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15707 // these string_compare variants as NEON register type for convenience so that the prototype of
15708 // string_compare can be shared with all variants.
15709
15710 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15711 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15712 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15713 pRegGov_P1 pgtmp2, rFlagsReg cr)
15714 %{
15715 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15716 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15717 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15718 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15719
15720 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15721 ins_encode %{
15722 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15723 __ string_compare($str1$$Register, $str2$$Register,
15724 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15725 $tmp1$$Register, $tmp2$$Register,
15726 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15727 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15728 StrIntrinsicNode::LL);
15729 %}
15730 ins_pipe(pipe_class_memory);
15731 %}
15732
15733 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15734 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15735 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15736 pRegGov_P1 pgtmp2, rFlagsReg cr)
15737 %{
15738 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15739 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15740 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15741 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15742
15743 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15744 ins_encode %{
15745 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15746 __ string_compare($str1$$Register, $str2$$Register,
15747 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15748 $tmp1$$Register, $tmp2$$Register,
15749 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15750 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15751 StrIntrinsicNode::LU);
15752 %}
15753 ins_pipe(pipe_class_memory);
15754 %}
15755
15756 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15757 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15758 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15759 pRegGov_P1 pgtmp2, rFlagsReg cr)
15760 %{
15761 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15762 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15763 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15764 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15765
15766 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15767 ins_encode %{
15768 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15769 __ string_compare($str1$$Register, $str2$$Register,
15770 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15771 $tmp1$$Register, $tmp2$$Register,
15772 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15773 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15774 StrIntrinsicNode::UL);
15775 %}
15776 ins_pipe(pipe_class_memory);
15777 %}
15778
15779 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15780 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15781 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15782 pRegGov_P1 pgtmp2, rFlagsReg cr)
15783 %{
15784 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15785 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15786 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15787 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15788
15789 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15790 ins_encode %{
15791 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15792 __ string_compare($str1$$Register, $str2$$Register,
15793 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15794 $tmp1$$Register, $tmp2$$Register,
15795 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15796 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15797 StrIntrinsicNode::UU);
15798 %}
15799 ins_pipe(pipe_class_memory);
15800 %}
15801
15802 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15803 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15804 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15805 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15806 %{
15807 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15808 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15809 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15810 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15811 TEMP vtmp0, TEMP vtmp1, KILL cr);
15812 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15813 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15814
15815 ins_encode %{
15816 __ string_indexof($str1$$Register, $str2$$Register,
15817 $cnt1$$Register, $cnt2$$Register,
15818 $tmp1$$Register, $tmp2$$Register,
15819 $tmp3$$Register, $tmp4$$Register,
15820 $tmp5$$Register, $tmp6$$Register,
15821 -1, $result$$Register, StrIntrinsicNode::UU);
15822 %}
15823 ins_pipe(pipe_class_memory);
15824 %}
15825
15826 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15827 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15828 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15829 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15830 %{
15831 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15832 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15833 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15834 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15835 TEMP vtmp0, TEMP vtmp1, KILL cr);
15836 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15837 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15838
15839 ins_encode %{
15840 __ string_indexof($str1$$Register, $str2$$Register,
15841 $cnt1$$Register, $cnt2$$Register,
15842 $tmp1$$Register, $tmp2$$Register,
15843 $tmp3$$Register, $tmp4$$Register,
15844 $tmp5$$Register, $tmp6$$Register,
15845 -1, $result$$Register, StrIntrinsicNode::LL);
15846 %}
15847 ins_pipe(pipe_class_memory);
15848 %}
15849
15850 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15851 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15852 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15853 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15854 %{
15855 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15856 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15857 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15858 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15859 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15860 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15861 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15862
15863 ins_encode %{
15864 __ string_indexof($str1$$Register, $str2$$Register,
15865 $cnt1$$Register, $cnt2$$Register,
15866 $tmp1$$Register, $tmp2$$Register,
15867 $tmp3$$Register, $tmp4$$Register,
15868 $tmp5$$Register, $tmp6$$Register,
15869 -1, $result$$Register, StrIntrinsicNode::UL);
15870 %}
15871 ins_pipe(pipe_class_memory);
15872 %}
15873
15874 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15875 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15876 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15877 %{
15878 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15879 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15880 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15881 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15882 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15883 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15884
15885 ins_encode %{
15886 int icnt2 = (int)$int_cnt2$$constant;
15887 __ string_indexof($str1$$Register, $str2$$Register,
15888 $cnt1$$Register, zr,
15889 $tmp1$$Register, $tmp2$$Register,
15890 $tmp3$$Register, $tmp4$$Register, zr, zr,
15891 icnt2, $result$$Register, StrIntrinsicNode::UU);
15892 %}
15893 ins_pipe(pipe_class_memory);
15894 %}
15895
15896 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15897 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15898 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15899 %{
15900 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15901 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15902 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15903 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15904 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15905 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15906
15907 ins_encode %{
15908 int icnt2 = (int)$int_cnt2$$constant;
15909 __ string_indexof($str1$$Register, $str2$$Register,
15910 $cnt1$$Register, zr,
15911 $tmp1$$Register, $tmp2$$Register,
15912 $tmp3$$Register, $tmp4$$Register, zr, zr,
15913 icnt2, $result$$Register, StrIntrinsicNode::LL);
15914 %}
15915 ins_pipe(pipe_class_memory);
15916 %}
15917
15918 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15919 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15920 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15921 %{
15922 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15923 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15924 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15925 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15926 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15927 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15928
15929 ins_encode %{
15930 int icnt2 = (int)$int_cnt2$$constant;
15931 __ string_indexof($str1$$Register, $str2$$Register,
15932 $cnt1$$Register, zr,
15933 $tmp1$$Register, $tmp2$$Register,
15934 $tmp3$$Register, $tmp4$$Register, zr, zr,
15935 icnt2, $result$$Register, StrIntrinsicNode::UL);
15936 %}
15937 ins_pipe(pipe_class_memory);
15938 %}
15939
15940 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15941 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15942 iRegINoSp tmp3, rFlagsReg cr)
15943 %{
15944 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15945 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15946 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15947 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15948
15949 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15950
15951 ins_encode %{
15952 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15953 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15954 $tmp3$$Register);
15955 %}
15956 ins_pipe(pipe_class_memory);
15957 %}
15958
15959 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15960 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15961 iRegINoSp tmp3, rFlagsReg cr)
15962 %{
15963 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15964 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15965 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15966 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15967
15968 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15969
15970 ins_encode %{
15971 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15972 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15973 $tmp3$$Register);
15974 %}
15975 ins_pipe(pipe_class_memory);
15976 %}
15977
15978 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15979 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15980 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15981 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
15982 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15983 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15984 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15985 ins_encode %{
15986 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15987 $result$$Register, $ztmp1$$FloatRegister,
15988 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15989 $ptmp$$PRegister, true /* isL */);
15990 %}
15991 ins_pipe(pipe_class_memory);
15992 %}
15993
15994 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15995 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15996 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15997 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
15998 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15999 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16000 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16001 ins_encode %{
16002 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16003 $result$$Register, $ztmp1$$FloatRegister,
16004 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16005 $ptmp$$PRegister, false /* isL */);
16006 %}
16007 ins_pipe(pipe_class_memory);
16008 %}
16009
16010 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16011 iRegI_R0 result, rFlagsReg cr)
16012 %{
16013 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16014 match(Set result (StrEquals (Binary str1 str2) cnt));
16015 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16016
16017 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16018 ins_encode %{
16019 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16020 __ string_equals($str1$$Register, $str2$$Register,
16021 $result$$Register, $cnt$$Register);
16022 %}
16023 ins_pipe(pipe_class_memory);
16024 %}
16025
16026 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16027 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16028 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16029 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16030 iRegP_R10 tmp, rFlagsReg cr)
16031 %{
16032 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16033 match(Set result (AryEq ary1 ary2));
16034 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16035 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16036 TEMP vtmp6, TEMP vtmp7, KILL cr);
16037
16038 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16039 ins_encode %{
16040 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16041 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16042 $result$$Register, $tmp$$Register, 1);
16043 if (tpc == nullptr) {
16044 ciEnv::current()->record_failure("CodeCache is full");
16045 return;
16046 }
16047 %}
16048 ins_pipe(pipe_class_memory);
16049 %}
16050
16051 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16052 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16053 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16054 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16055 iRegP_R10 tmp, rFlagsReg cr)
16056 %{
16057 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16058 match(Set result (AryEq ary1 ary2));
16059 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16060 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16061 TEMP vtmp6, TEMP vtmp7, KILL cr);
16062
16063 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16064 ins_encode %{
16065 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16066 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16067 $result$$Register, $tmp$$Register, 2);
16068 if (tpc == nullptr) {
16069 ciEnv::current()->record_failure("CodeCache is full");
16070 return;
16071 }
16072 %}
16073 ins_pipe(pipe_class_memory);
16074 %}
16075
16076 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16077 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16078 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16079 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16080 %{
16081 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16082 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16083 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16084
16085 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16086 ins_encode %{
16087 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16088 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16089 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16090 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16091 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16092 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16093 (BasicType)$basic_type$$constant);
16094 if (tpc == nullptr) {
16095 ciEnv::current()->record_failure("CodeCache is full");
16096 return;
16097 }
16098 %}
16099 ins_pipe(pipe_class_memory);
16100 %}
16101
16102 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16103 %{
16104 match(Set result (CountPositives ary1 len));
16105 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16106 format %{ "count positives byte[] $ary1,$len -> $result" %}
16107 ins_encode %{
16108 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16109 if (tpc == nullptr) {
16110 ciEnv::current()->record_failure("CodeCache is full");
16111 return;
16112 }
16113 %}
16114 ins_pipe( pipe_slow );
16115 %}
16116
16117 // fast char[] to byte[] compression
16118 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16119 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16120 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16121 iRegI_R0 result, rFlagsReg cr)
16122 %{
16123 match(Set result (StrCompressedCopy src (Binary dst len)));
16124 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16125 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16126
16127 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16128 ins_encode %{
16129 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16130 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16131 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16132 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16133 %}
16134 ins_pipe(pipe_slow);
16135 %}
16136
16137 // fast byte[] to char[] inflation
16138 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16139 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16140 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16141 %{
16142 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16143 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16144 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16145 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16146
16147 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16148 ins_encode %{
16149 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16150 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16151 $vtmp2$$FloatRegister, $tmp$$Register);
16152 if (tpc == nullptr) {
16153 ciEnv::current()->record_failure("CodeCache is full");
16154 return;
16155 }
16156 %}
16157 ins_pipe(pipe_class_memory);
16158 %}
16159
16160 // encode char[] to byte[] in ISO_8859_1
16161 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16162 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16163 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16164 iRegI_R0 result, rFlagsReg cr)
16165 %{
16166 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16167 match(Set result (EncodeISOArray src (Binary dst len)));
16168 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16169 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16170
16171 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16172 ins_encode %{
16173 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16174 $result$$Register, false,
16175 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16176 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16177 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16178 %}
16179 ins_pipe(pipe_class_memory);
16180 %}
16181
16182 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16183 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16184 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16185 iRegI_R0 result, rFlagsReg cr)
16186 %{
16187 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16188 match(Set result (EncodeISOArray src (Binary dst len)));
16189 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16190 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16191
16192 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16193 ins_encode %{
16194 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16195 $result$$Register, true,
16196 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16197 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16198 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16199 %}
16200 ins_pipe(pipe_class_memory);
16201 %}
16202
16203 //----------------------------- CompressBits/ExpandBits ------------------------
16204
16205 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16206 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16207 match(Set dst (CompressBits src mask));
16208 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16209 format %{ "mov $tsrc, $src\n\t"
16210 "mov $tmask, $mask\n\t"
16211 "bext $tdst, $tsrc, $tmask\n\t"
16212 "mov $dst, $tdst"
16213 %}
16214 ins_encode %{
16215 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16216 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16217 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16218 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16219 %}
16220 ins_pipe(pipe_slow);
16221 %}
16222
16223 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16224 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16225 match(Set dst (CompressBits (LoadI mem) mask));
16226 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16227 format %{ "ldrs $tsrc, $mem\n\t"
16228 "ldrs $tmask, $mask\n\t"
16229 "bext $tdst, $tsrc, $tmask\n\t"
16230 "mov $dst, $tdst"
16231 %}
16232 ins_encode %{
16233 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16234 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16235 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16236 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16237 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16238 %}
16239 ins_pipe(pipe_slow);
16240 %}
16241
16242 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16243 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16244 match(Set dst (CompressBits src mask));
16245 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16246 format %{ "mov $tsrc, $src\n\t"
16247 "mov $tmask, $mask\n\t"
16248 "bext $tdst, $tsrc, $tmask\n\t"
16249 "mov $dst, $tdst"
16250 %}
16251 ins_encode %{
16252 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16253 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16254 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16255 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16256 %}
16257 ins_pipe(pipe_slow);
16258 %}
16259
16260 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16261 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16262 match(Set dst (CompressBits (LoadL mem) mask));
16263 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16264 format %{ "ldrd $tsrc, $mem\n\t"
16265 "ldrd $tmask, $mask\n\t"
16266 "bext $tdst, $tsrc, $tmask\n\t"
16267 "mov $dst, $tdst"
16268 %}
16269 ins_encode %{
16270 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16271 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16272 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16273 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16274 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16275 %}
16276 ins_pipe(pipe_slow);
16277 %}
16278
16279 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16280 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16281 match(Set dst (ExpandBits src mask));
16282 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16283 format %{ "mov $tsrc, $src\n\t"
16284 "mov $tmask, $mask\n\t"
16285 "bdep $tdst, $tsrc, $tmask\n\t"
16286 "mov $dst, $tdst"
16287 %}
16288 ins_encode %{
16289 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16290 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16291 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16292 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16293 %}
16294 ins_pipe(pipe_slow);
16295 %}
16296
16297 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16298 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16299 match(Set dst (ExpandBits (LoadI mem) mask));
16300 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16301 format %{ "ldrs $tsrc, $mem\n\t"
16302 "ldrs $tmask, $mask\n\t"
16303 "bdep $tdst, $tsrc, $tmask\n\t"
16304 "mov $dst, $tdst"
16305 %}
16306 ins_encode %{
16307 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16308 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16309 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16310 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16311 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16312 %}
16313 ins_pipe(pipe_slow);
16314 %}
16315
16316 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16317 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16318 match(Set dst (ExpandBits src mask));
16319 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16320 format %{ "mov $tsrc, $src\n\t"
16321 "mov $tmask, $mask\n\t"
16322 "bdep $tdst, $tsrc, $tmask\n\t"
16323 "mov $dst, $tdst"
16324 %}
16325 ins_encode %{
16326 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16327 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16328 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16329 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16330 %}
16331 ins_pipe(pipe_slow);
16332 %}
16333
16334
16335 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16336 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16337 match(Set dst (ExpandBits (LoadL mem) mask));
16338 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16339 format %{ "ldrd $tsrc, $mem\n\t"
16340 "ldrd $tmask, $mask\n\t"
16341 "bdep $tdst, $tsrc, $tmask\n\t"
16342 "mov $dst, $tdst"
16343 %}
16344 ins_encode %{
16345 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16346 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16347 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16348 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16349 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16350 %}
16351 ins_pipe(pipe_slow);
16352 %}
16353
16354 //----------------------------- Reinterpret ----------------------------------
16355 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16356 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16357 match(Set dst (ReinterpretHF2S src));
16358 format %{ "reinterpretHF2S $dst, $src" %}
16359 ins_encode %{
16360 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16361 %}
16362 ins_pipe(pipe_slow);
16363 %}
16364
16365 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16366 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16367 match(Set dst (ReinterpretS2HF src));
16368 format %{ "reinterpretS2HF $dst, $src" %}
16369 ins_encode %{
16370 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16371 %}
16372 ins_pipe(pipe_slow);
16373 %}
16374
16375 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16376 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16377 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16378 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16379 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16380 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16381 // can be omitted in this pattern, resulting in -
16382 // fcvt $dst, $src // Convert float to half-precision float
16383 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16384 %{
16385 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16386 format %{ "convF2HFAndS2HF $dst, $src" %}
16387 ins_encode %{
16388 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16389 %}
16390 ins_pipe(pipe_slow);
16391 %}
16392
16393 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16394 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16395 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16396 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16397 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16398 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16399 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16400 // resulting in -
16401 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16402 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16403 %{
16404 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16405 format %{ "convHF2SAndHF2F $dst, $src" %}
16406 ins_encode %{
16407 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16408 %}
16409 ins_pipe(pipe_slow);
16410 %}
16411
16412 // ============================================================================
16413 // This name is KNOWN by the ADLC and cannot be changed.
16414 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16415 // for this guy.
16416 instruct tlsLoadP(thread_RegP dst)
16417 %{
16418 match(Set dst (ThreadLocal));
16419
16420 ins_cost(0);
16421
16422 format %{ " -- \t// $dst=Thread::current(), empty" %}
16423
16424 size(0);
16425
16426 ins_encode( /*empty*/ );
16427
16428 ins_pipe(pipe_class_empty);
16429 %}
16430
16431 //----------PEEPHOLE RULES-----------------------------------------------------
16432 // These must follow all instruction definitions as they use the names
16433 // defined in the instructions definitions.
16434 //
16435 // peepmatch ( root_instr_name [preceding_instruction]* );
16436 //
16437 // peepconstraint %{
16438 // (instruction_number.operand_name relational_op instruction_number.operand_name
16439 // [, ...] );
16440 // // instruction numbers are zero-based using left to right order in peepmatch
16441 //
16442 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16443 // // provide an instruction_number.operand_name for each operand that appears
16444 // // in the replacement instruction's match rule
16445 //
16446 // ---------VM FLAGS---------------------------------------------------------
16447 //
16448 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16449 //
16450 // Each peephole rule is given an identifying number starting with zero and
16451 // increasing by one in the order seen by the parser. An individual peephole
16452 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16453 // on the command-line.
16454 //
16455 // ---------CURRENT LIMITATIONS----------------------------------------------
16456 //
16457 // Only match adjacent instructions in same basic block
16458 // Only equality constraints
16459 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16460 // Only one replacement instruction
16461 //
16462 // ---------EXAMPLE----------------------------------------------------------
16463 //
16464 // // pertinent parts of existing instructions in architecture description
16465 // instruct movI(iRegINoSp dst, iRegI src)
16466 // %{
16467 // match(Set dst (CopyI src));
16468 // %}
16469 //
16470 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16471 // %{
16472 // match(Set dst (AddI dst src));
16473 // effect(KILL cr);
16474 // %}
16475 //
16476 // // Change (inc mov) to lea
16477 // peephole %{
16478 // // increment preceded by register-register move
16479 // peepmatch ( incI_iReg movI );
16480 // // require that the destination register of the increment
16481 // // match the destination register of the move
16482 // peepconstraint ( 0.dst == 1.dst );
16483 // // construct a replacement instruction that sets
16484 // // the destination to ( move's source register + one )
16485 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16486 // %}
16487 //
16488
16489 // Implementation no longer uses movX instructions since
16490 // machine-independent system no longer uses CopyX nodes.
16491 //
16492 // peephole
16493 // %{
16494 // peepmatch (incI_iReg movI);
16495 // peepconstraint (0.dst == 1.dst);
16496 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16497 // %}
16498
16499 // peephole
16500 // %{
16501 // peepmatch (decI_iReg movI);
16502 // peepconstraint (0.dst == 1.dst);
16503 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16504 // %}
16505
16506 // peephole
16507 // %{
16508 // peepmatch (addI_iReg_imm movI);
16509 // peepconstraint (0.dst == 1.dst);
16510 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16511 // %}
16512
16513 // peephole
16514 // %{
16515 // peepmatch (incL_iReg movL);
16516 // peepconstraint (0.dst == 1.dst);
16517 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16518 // %}
16519
16520 // peephole
16521 // %{
16522 // peepmatch (decL_iReg movL);
16523 // peepconstraint (0.dst == 1.dst);
16524 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16525 // %}
16526
16527 // peephole
16528 // %{
16529 // peepmatch (addL_iReg_imm movL);
16530 // peepconstraint (0.dst == 1.dst);
16531 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16532 // %}
16533
16534 // peephole
16535 // %{
16536 // peepmatch (addP_iReg_imm movP);
16537 // peepconstraint (0.dst == 1.dst);
16538 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16539 // %}
16540
16541 // // Change load of spilled value to only a spill
16542 // instruct storeI(memory mem, iRegI src)
16543 // %{
16544 // match(Set mem (StoreI mem src));
16545 // %}
16546 //
16547 // instruct loadI(iRegINoSp dst, memory mem)
16548 // %{
16549 // match(Set dst (LoadI mem));
16550 // %}
16551 //
16552
16553 //----------SMARTSPILL RULES---------------------------------------------------
16554 // These must follow all instruction definitions as they use the names
16555 // defined in the instructions definitions.
16556
16557 // Local Variables:
16558 // mode: c++
16559 // End: