1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return MacroAssembler::max_trampoline_stub_size();
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 5; // metadata; call dest; trampoline address; trampoline destination; trampoline_owner_metadata
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else {
1694 return 6 * NativeInstruction::instruction_size;
1695 }
1696 }
1697
1698 //=============================================================================
1699
1700 #ifndef PRODUCT
1701 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1702 st->print("BREAKPOINT");
1703 }
1704 #endif
1705
1706 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1707 __ brk(0);
1708 }
1709
1710 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1711 return MachNode::size(ra_);
1712 }
1713
1714 //=============================================================================
1715
1716 #ifndef PRODUCT
1717 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1718 st->print("nop \t# %d bytes pad for loops and calls", _count);
1719 }
1720 #endif
1721
1722 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1723 for (int i = 0; i < _count; i++) {
1724 __ nop();
1725 }
1726 }
1727
1728 uint MachNopNode::size(PhaseRegAlloc*) const {
1729 return _count * NativeInstruction::instruction_size;
1730 }
1731
1732 //=============================================================================
1733 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1734
1735 int ConstantTable::calculate_table_base_offset() const {
1736 return 0; // absolute addressing, no offset
1737 }
1738
1739 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1740 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1741 ShouldNotReachHere();
1742 }
1743
1744 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1745 // Empty encoding
1746 }
1747
1748 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1749 return 0;
1750 }
1751
1752 #ifndef PRODUCT
1753 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1754 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1755 }
1756 #endif
1757
1758 #ifndef PRODUCT
1759 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1760 Compile* C = ra_->C;
1761
1762 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1763
1764 if (C->output()->need_stack_bang(framesize))
1765 st->print("# stack bang size=%d\n\t", framesize);
1766
1767 if (VM_Version::use_rop_protection()) {
1768 st->print("ldr zr, [lr]\n\t");
1769 st->print("paciaz\n\t");
1770 }
1771 if (framesize < ((1 << 9) + 2 * wordSize)) {
1772 st->print("sub sp, sp, #%d\n\t", framesize);
1773 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1774 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1775 } else {
1776 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1777 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1778 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1779 st->print("sub sp, sp, rscratch1");
1780 }
1781 if (C->stub_function() == nullptr) {
1782 st->print("\n\t");
1783 st->print("ldr rscratch1, [guard]\n\t");
1784 st->print("dmb ishld\n\t");
1785 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1786 st->print("cmp rscratch1, rscratch2\n\t");
1787 st->print("b.eq skip");
1788 st->print("\n\t");
1789 st->print("blr #nmethod_entry_barrier_stub\n\t");
1790 st->print("b skip\n\t");
1791 st->print("guard: int\n\t");
1792 st->print("\n\t");
1793 st->print("skip:\n\t");
1794 }
1795 }
1796 #endif
1797
1798 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1799 Compile* C = ra_->C;
1800
1801 // n.b. frame size includes space for return pc and rfp
1802 const int framesize = C->output()->frame_size_in_bytes();
1803
1804 if (C->clinit_barrier_on_entry()) {
1805 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1806
1807 Label L_skip_barrier;
1808
1809 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1810 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1811 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1812 __ bind(L_skip_barrier);
1813 }
1814
1815 if (C->max_vector_size() > 0) {
1816 __ reinitialize_ptrue();
1817 }
1818
1819 int bangsize = C->output()->bang_size_in_bytes();
1820 if (C->output()->need_stack_bang(bangsize))
1821 __ generate_stack_overflow_check(bangsize);
1822
1823 __ build_frame(framesize);
1824
1825 if (C->stub_function() == nullptr) {
1826 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1827 // Dummy labels for just measuring the code size
1828 Label dummy_slow_path;
1829 Label dummy_continuation;
1830 Label dummy_guard;
1831 Label* slow_path = &dummy_slow_path;
1832 Label* continuation = &dummy_continuation;
1833 Label* guard = &dummy_guard;
1834 if (!Compile::current()->output()->in_scratch_emit_size()) {
1835 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1836 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1837 Compile::current()->output()->add_stub(stub);
1838 slow_path = &stub->entry();
1839 continuation = &stub->continuation();
1840 guard = &stub->guard();
1841 }
1842 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1843 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1844 }
1845
1846 if (VerifyStackAtCalls) {
1847 Unimplemented();
1848 }
1849
1850 C->output()->set_frame_complete(__ offset());
1851
1852 if (C->has_mach_constant_base_node()) {
1853 // NOTE: We set the table base offset here because users might be
1854 // emitted before MachConstantBaseNode.
1855 ConstantTable& constant_table = C->output()->constant_table();
1856 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1857 }
1858 }
1859
1860 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1861 {
1862 return MachNode::size(ra_); // too many variables; just compute it
1863 // the hard way
1864 }
1865
1866 int MachPrologNode::reloc() const
1867 {
1868 return 0;
1869 }
1870
1871 //=============================================================================
1872
1873 #ifndef PRODUCT
1874 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 st->print("# pop frame %d\n\t",framesize);
1879
1880 if (framesize == 0) {
1881 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1882 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1883 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1884 st->print("add sp, sp, #%d\n\t", framesize);
1885 } else {
1886 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1887 st->print("add sp, sp, rscratch1\n\t");
1888 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1889 }
1890 if (VM_Version::use_rop_protection()) {
1891 st->print("autiaz\n\t");
1892 st->print("ldr zr, [lr]\n\t");
1893 }
1894
1895 if (do_polling() && C->is_method_compilation()) {
1896 st->print("# test polling word\n\t");
1897 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1898 st->print("cmp sp, rscratch1\n\t");
1899 st->print("bhi #slow_path");
1900 }
1901 }
1902 #endif
1903
1904 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1905 Compile* C = ra_->C;
1906 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1907
1908 __ remove_frame(framesize);
1909
1910 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1911 __ reserved_stack_check();
1912 }
1913
1914 if (do_polling() && C->is_method_compilation()) {
1915 Label dummy_label;
1916 Label* code_stub = &dummy_label;
1917 if (!C->output()->in_scratch_emit_size()) {
1918 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1919 C->output()->add_stub(stub);
1920 code_stub = &stub->entry();
1921 }
1922 __ relocate(relocInfo::poll_return_type);
1923 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1924 }
1925 }
1926
1927 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1928 // Variable size. Determine dynamically.
1929 return MachNode::size(ra_);
1930 }
1931
1932 int MachEpilogNode::reloc() const {
1933 // Return number of relocatable values contained in this instruction.
1934 return 1; // 1 for polling page.
1935 }
1936
1937 const Pipeline * MachEpilogNode::pipeline() const {
1938 return MachNode::pipeline_class();
1939 }
1940
1941 //=============================================================================
1942
1943 static enum RC rc_class(OptoReg::Name reg) {
1944
1945 if (reg == OptoReg::Bad) {
1946 return rc_bad;
1947 }
1948
1949 // we have 32 int registers * 2 halves
1950 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1951
1952 if (reg < slots_of_int_registers) {
1953 return rc_int;
1954 }
1955
1956 // we have 32 float register * 8 halves
1957 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1958 if (reg < slots_of_int_registers + slots_of_float_registers) {
1959 return rc_float;
1960 }
1961
1962 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1963 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1964 return rc_predicate;
1965 }
1966
1967 // Between predicate regs & stack is the flags.
1968 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1969
1970 return rc_stack;
1971 }
1972
1973 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1974 Compile* C = ra_->C;
1975
1976 // Get registers to move.
1977 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1978 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1979 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1980 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1981
1982 enum RC src_hi_rc = rc_class(src_hi);
1983 enum RC src_lo_rc = rc_class(src_lo);
1984 enum RC dst_hi_rc = rc_class(dst_hi);
1985 enum RC dst_lo_rc = rc_class(dst_lo);
1986
1987 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1988
1989 if (src_hi != OptoReg::Bad && !bottom_type()->isa_pvectmask()) {
1990 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1991 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1992 "expected aligned-adjacent pairs");
1993 }
1994
1995 if (src_lo == dst_lo && src_hi == dst_hi) {
1996 return 0; // Self copy, no move.
1997 }
1998
1999 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2000 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2001 int src_offset = ra_->reg2offset(src_lo);
2002 int dst_offset = ra_->reg2offset(dst_lo);
2003
2004 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
2005 uint ireg = ideal_reg();
2006 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
2007 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
2008 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
2009 if (ireg == Op_VecA && masm) {
2010 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2011 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2012 // stack->stack
2013 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2016 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2017 sve_vector_reg_size_in_bytes);
2018 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2019 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2020 sve_vector_reg_size_in_bytes);
2021 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2022 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2023 as_FloatRegister(Matcher::_regEncode[src_lo]),
2024 as_FloatRegister(Matcher::_regEncode[src_lo]));
2025 } else {
2026 ShouldNotReachHere();
2027 }
2028 } else if (masm) {
2029 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2030 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2031 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2032 // stack->stack
2033 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2034 if (ireg == Op_VecD) {
2035 __ unspill(rscratch1, true, src_offset);
2036 __ spill(rscratch1, true, dst_offset);
2037 } else {
2038 __ spill_copy128(src_offset, dst_offset);
2039 }
2040 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2041 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2042 ireg == Op_VecD ? __ T8B : __ T16B,
2043 as_FloatRegister(Matcher::_regEncode[src_lo]));
2044 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2045 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2046 ireg == Op_VecD ? __ D : __ Q,
2047 ra_->reg2offset(dst_lo));
2048 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2049 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2050 ireg == Op_VecD ? __ D : __ Q,
2051 ra_->reg2offset(src_lo));
2052 } else {
2053 ShouldNotReachHere();
2054 }
2055 }
2056 } else if (masm) {
2057 switch (src_lo_rc) {
2058 case rc_int:
2059 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2060 if (is64) {
2061 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2062 as_Register(Matcher::_regEncode[src_lo]));
2063 } else {
2064 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2065 as_Register(Matcher::_regEncode[src_lo]));
2066 }
2067 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2068 if (is64) {
2069 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2070 as_Register(Matcher::_regEncode[src_lo]));
2071 } else {
2072 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2073 as_Register(Matcher::_regEncode[src_lo]));
2074 }
2075 } else { // gpr --> stack spill
2076 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2077 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2078 }
2079 break;
2080 case rc_float:
2081 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2082 if (is64) {
2083 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2084 as_FloatRegister(Matcher::_regEncode[src_lo]));
2085 } else {
2086 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2087 as_FloatRegister(Matcher::_regEncode[src_lo]));
2088 }
2089 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2090 if (is64) {
2091 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2092 as_FloatRegister(Matcher::_regEncode[src_lo]));
2093 } else {
2094 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2095 as_FloatRegister(Matcher::_regEncode[src_lo]));
2096 }
2097 } else { // fpr --> stack spill
2098 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2099 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2100 is64 ? __ D : __ S, dst_offset);
2101 }
2102 break;
2103 case rc_stack:
2104 if (dst_lo_rc == rc_int) { // stack --> gpr load
2105 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2106 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2107 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2108 is64 ? __ D : __ S, src_offset);
2109 } else if (dst_lo_rc == rc_predicate) {
2110 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2111 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2112 } else { // stack --> stack copy
2113 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2114 if (ideal_reg() == Op_RegVectMask) {
2115 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2116 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2117 } else {
2118 __ unspill(rscratch1, is64, src_offset);
2119 __ spill(rscratch1, is64, dst_offset);
2120 }
2121 }
2122 break;
2123 case rc_predicate:
2124 if (dst_lo_rc == rc_predicate) {
2125 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2126 } else if (dst_lo_rc == rc_stack) {
2127 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2128 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2129 } else {
2130 assert(false, "bad src and dst rc_class combination.");
2131 ShouldNotReachHere();
2132 }
2133 break;
2134 default:
2135 assert(false, "bad rc_class for spill");
2136 ShouldNotReachHere();
2137 }
2138 }
2139
2140 if (st) {
2141 st->print("spill ");
2142 if (src_lo_rc == rc_stack) {
2143 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2144 } else {
2145 st->print("%s -> ", Matcher::regName[src_lo]);
2146 }
2147 if (dst_lo_rc == rc_stack) {
2148 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2149 } else {
2150 st->print("%s", Matcher::regName[dst_lo]);
2151 }
2152 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
2153 int vsize = 0;
2154 switch (ideal_reg()) {
2155 case Op_VecD:
2156 vsize = 64;
2157 break;
2158 case Op_VecX:
2159 vsize = 128;
2160 break;
2161 case Op_VecA:
2162 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2163 break;
2164 default:
2165 assert(false, "bad register type for spill");
2166 ShouldNotReachHere();
2167 }
2168 st->print("\t# vector spill size = %d", vsize);
2169 } else if (ideal_reg() == Op_RegVectMask) {
2170 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2171 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2172 st->print("\t# predicate spill size = %d", vsize);
2173 } else {
2174 st->print("\t# spill size = %d", is64 ? 64 : 32);
2175 }
2176 }
2177
2178 return 0;
2179
2180 }
2181
2182 #ifndef PRODUCT
2183 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2184 if (!ra_)
2185 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2186 else
2187 implementation(nullptr, ra_, false, st);
2188 }
2189 #endif
2190
2191 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2192 implementation(masm, ra_, false, nullptr);
2193 }
2194
2195 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2196 return MachNode::size(ra_);
2197 }
2198
2199 //=============================================================================
2200
2201 #ifndef PRODUCT
2202 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2203 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2204 int reg = ra_->get_reg_first(this);
2205 st->print("add %s, rsp, #%d]\t# box lock",
2206 Matcher::regName[reg], offset);
2207 }
2208 #endif
2209
2210 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2211 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2212 int reg = ra_->get_encode(this);
2213
2214 // This add will handle any 24-bit signed offset. 24 bits allows an
2215 // 8 megabyte stack frame.
2216 __ add(as_Register(reg), sp, offset);
2217 }
2218
2219 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2220 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2221 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2222
2223 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2224 return NativeInstruction::instruction_size;
2225 } else {
2226 return 2 * NativeInstruction::instruction_size;
2227 }
2228 }
2229
2230 //=============================================================================
2231
2232 #ifndef PRODUCT
2233 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2234 {
2235 st->print_cr("# MachUEPNode");
2236 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2237 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2238 st->print_cr("\tcmpw rscratch1, r10");
2239 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2240 }
2241 #endif
2242
2243 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2244 {
2245 __ ic_check(InteriorEntryAlignment);
2246 }
2247
2248 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2249 {
2250 return MachNode::size(ra_);
2251 }
2252
2253 // REQUIRED EMIT CODE
2254
2255 //=============================================================================
2256
2257 // Emit deopt handler code.
2258 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2259 {
2260 // Note that the code buffer's insts_mark is always relative to insts.
2261 // That's why we must use the macroassembler to generate a handler.
2262 address base = __ start_a_stub(size_deopt_handler());
2263 if (base == nullptr) {
2264 ciEnv::current()->record_failure("CodeCache is full");
2265 return 0; // CodeBuffer::expand failed
2266 }
2267
2268 int offset = __ offset();
2269 Label start;
2270 __ bind(start);
2271 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2272
2273 int entry_offset = __ offset();
2274 __ b(start);
2275
2276 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2277 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2278 "out of bounds read in post-call NOP check");
2279 __ end_a_stub();
2280 return entry_offset;
2281 }
2282
2283 // REQUIRED MATCHER CODE
2284
2285 //=============================================================================
2286
2287 bool Matcher::match_rule_supported(int opcode) {
2288 if (!has_match_rule(opcode))
2289 return false;
2290
2291 switch (opcode) {
2292 case Op_OnSpinWait:
2293 return VM_Version::supports_on_spin_wait();
2294 case Op_CacheWB:
2295 case Op_CacheWBPreSync:
2296 case Op_CacheWBPostSync:
2297 if (!VM_Version::supports_data_cache_line_flush()) {
2298 return false;
2299 }
2300 break;
2301 case Op_ExpandBits:
2302 case Op_CompressBits:
2303 if (!VM_Version::supports_svebitperm()) {
2304 return false;
2305 }
2306 break;
2307 case Op_FmaF:
2308 case Op_FmaD:
2309 case Op_FmaVF:
2310 case Op_FmaVD:
2311 if (!UseFMA) {
2312 return false;
2313 }
2314 break;
2315 case Op_FmaHF:
2316 // UseFMA flag also needs to be checked along with FEAT_FP16
2317 if (!UseFMA || !is_feat_fp16_supported()) {
2318 return false;
2319 }
2320 break;
2321 case Op_AddHF:
2322 case Op_SubHF:
2323 case Op_MulHF:
2324 case Op_DivHF:
2325 case Op_MinHF:
2326 case Op_MaxHF:
2327 case Op_SqrtHF:
2328 // Half-precision floating point scalar operations require FEAT_FP16
2329 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2330 // features are supported.
2331 if (!is_feat_fp16_supported()) {
2332 return false;
2333 }
2334 break;
2335 }
2336
2337 return true; // Per default match rules are supported.
2338 }
2339
2340 const RegMask* Matcher::predicate_reg_mask(void) {
2341 return &_PR_REG_mask;
2342 }
2343
2344 bool Matcher::supports_vector_calling_convention(void) {
2345 return EnableVectorSupport;
2346 }
2347
2348 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2349 assert(EnableVectorSupport, "sanity");
2350 int lo = V0_num;
2351 int hi = V0_H_num;
2352 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2353 hi = V0_K_num;
2354 }
2355 return OptoRegPair(hi, lo);
2356 }
2357
2358 // Is this branch offset short enough that a short branch can be used?
2359 //
2360 // NOTE: If the platform does not provide any short branch variants, then
2361 // this method should return false for offset 0.
2362 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2363 // The passed offset is relative to address of the branch.
2364
2365 return (-32768 <= offset && offset < 32768);
2366 }
2367
2368 // Vector width in bytes.
2369 int Matcher::vector_width_in_bytes(BasicType bt) {
2370 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2371 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2372 // Minimum 2 values in vector
2373 if (size < 2*type2aelembytes(bt)) size = 0;
2374 // But never < 4
2375 if (size < 4) size = 0;
2376 return size;
2377 }
2378
2379 // Limits on vector size (number of elements) loaded into vector.
2380 int Matcher::max_vector_size(const BasicType bt) {
2381 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2382 }
2383
2384 int Matcher::min_vector_size(const BasicType bt) {
2385 // Usually, the shortest vector length supported by AArch64 ISA and
2386 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2387 // vectors in a few special cases.
2388 int size;
2389 switch(bt) {
2390 case T_BOOLEAN:
2391 // Load/store a vector mask with only 2 elements for vector types
2392 // such as "2I/2F/2L/2D".
2393 size = 2;
2394 break;
2395 case T_BYTE:
2396 // Generate a "4B" vector, to support vector cast between "8B/16B"
2397 // and "4S/4I/4L/4F/4D".
2398 size = 4;
2399 break;
2400 case T_SHORT:
2401 // Generate a "2S" vector, to support vector cast between "4S/8S"
2402 // and "2I/2L/2F/2D".
2403 size = 2;
2404 break;
2405 default:
2406 // Limit the min vector length to 64-bit.
2407 size = 8 / type2aelembytes(bt);
2408 // The number of elements in a vector should be at least 2.
2409 size = MAX2(size, 2);
2410 }
2411
2412 int max_size = max_vector_size(bt);
2413 return MIN2(size, max_size);
2414 }
2415
2416 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2417 return Matcher::max_vector_size(bt);
2418 }
2419
2420 // Actual max scalable vector register length.
2421 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2422 return Matcher::max_vector_size(bt);
2423 }
2424
2425 // Vector ideal reg.
2426 uint Matcher::vector_ideal_reg(int len) {
2427 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2428 return Op_VecA;
2429 }
2430 switch(len) {
2431 // For 16-bit/32-bit mask vector, reuse VecD.
2432 case 2:
2433 case 4:
2434 case 8: return Op_VecD;
2435 case 16: return Op_VecX;
2436 }
2437 ShouldNotReachHere();
2438 return 0;
2439 }
2440
2441 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2442 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2443 switch (ideal_reg) {
2444 case Op_VecA: return new vecAOper();
2445 case Op_VecD: return new vecDOper();
2446 case Op_VecX: return new vecXOper();
2447 }
2448 ShouldNotReachHere();
2449 return nullptr;
2450 }
2451
2452 bool Matcher::is_reg2reg_move(MachNode* m) {
2453 return false;
2454 }
2455
2456 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2457 return false;
2458 }
2459
2460 bool Matcher::is_generic_vector(MachOper* opnd) {
2461 return opnd->opcode() == VREG;
2462 }
2463
2464 #ifdef ASSERT
2465 // Return whether or not this register is ever used as an argument.
2466 bool Matcher::can_be_java_arg(int reg)
2467 {
2468 return
2469 reg == R0_num || reg == R0_H_num ||
2470 reg == R1_num || reg == R1_H_num ||
2471 reg == R2_num || reg == R2_H_num ||
2472 reg == R3_num || reg == R3_H_num ||
2473 reg == R4_num || reg == R4_H_num ||
2474 reg == R5_num || reg == R5_H_num ||
2475 reg == R6_num || reg == R6_H_num ||
2476 reg == R7_num || reg == R7_H_num ||
2477 reg == V0_num || reg == V0_H_num ||
2478 reg == V1_num || reg == V1_H_num ||
2479 reg == V2_num || reg == V2_H_num ||
2480 reg == V3_num || reg == V3_H_num ||
2481 reg == V4_num || reg == V4_H_num ||
2482 reg == V5_num || reg == V5_H_num ||
2483 reg == V6_num || reg == V6_H_num ||
2484 reg == V7_num || reg == V7_H_num;
2485 }
2486 #endif
2487
2488 uint Matcher::int_pressure_limit()
2489 {
2490 // JDK-8183543: When taking the number of available registers as int
2491 // register pressure threshold, the jtreg test:
2492 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2493 // failed due to C2 compilation failure with
2494 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2495 //
2496 // A derived pointer is live at CallNode and then is flagged by RA
2497 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2498 // derived pointers and lastly fail to spill after reaching maximum
2499 // number of iterations. Lowering the default pressure threshold to
2500 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2501 // a high register pressure area of the code so that split_DEF can
2502 // generate DefinitionSpillCopy for the derived pointer.
2503 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2504 if (!PreserveFramePointer) {
2505 // When PreserveFramePointer is off, frame pointer is allocatable,
2506 // but different from other SOC registers, it is excluded from
2507 // fatproj's mask because its save type is No-Save. Decrease 1 to
2508 // ensure high pressure at fatproj when PreserveFramePointer is off.
2509 // See check_pressure_at_fatproj().
2510 default_int_pressure_threshold--;
2511 }
2512 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2513 }
2514
2515 uint Matcher::float_pressure_limit()
2516 {
2517 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2518 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2519 }
2520
2521 const RegMask& Matcher::divI_proj_mask() {
2522 ShouldNotReachHere();
2523 return RegMask::EMPTY;
2524 }
2525
2526 // Register for MODI projection of divmodI.
2527 const RegMask& Matcher::modI_proj_mask() {
2528 ShouldNotReachHere();
2529 return RegMask::EMPTY;
2530 }
2531
2532 // Register for DIVL projection of divmodL.
2533 const RegMask& Matcher::divL_proj_mask() {
2534 ShouldNotReachHere();
2535 return RegMask::EMPTY;
2536 }
2537
2538 // Register for MODL projection of divmodL.
2539 const RegMask& Matcher::modL_proj_mask() {
2540 ShouldNotReachHere();
2541 return RegMask::EMPTY;
2542 }
2543
2544 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2545 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2546 Node* u = addp->fast_out(i);
2547 if (u->is_LoadStore()) {
2548 // On AArch64, LoadStoreNodes (i.e. compare and swap
2549 // instructions) only take register indirect as an operand, so
2550 // any attempt to use an AddPNode as an input to a LoadStoreNode
2551 // must fail.
2552 return false;
2553 }
2554 if (u->is_Mem()) {
2555 int opsize = u->as_Mem()->memory_size();
2556 assert(opsize > 0, "unexpected memory operand size");
2557 if (u->as_Mem()->memory_size() != (1<<shift)) {
2558 return false;
2559 }
2560 }
2561 }
2562 return true;
2563 }
2564
2565 // Convert BoolTest condition to Assembler condition.
2566 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2567 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2568 Assembler::Condition result;
2569 switch(cond) {
2570 case BoolTest::eq:
2571 result = Assembler::EQ; break;
2572 case BoolTest::ne:
2573 result = Assembler::NE; break;
2574 case BoolTest::le:
2575 result = Assembler::LE; break;
2576 case BoolTest::ge:
2577 result = Assembler::GE; break;
2578 case BoolTest::lt:
2579 result = Assembler::LT; break;
2580 case BoolTest::gt:
2581 result = Assembler::GT; break;
2582 case BoolTest::ule:
2583 result = Assembler::LS; break;
2584 case BoolTest::uge:
2585 result = Assembler::HS; break;
2586 case BoolTest::ult:
2587 result = Assembler::LO; break;
2588 case BoolTest::ugt:
2589 result = Assembler::HI; break;
2590 case BoolTest::overflow:
2591 result = Assembler::VS; break;
2592 case BoolTest::no_overflow:
2593 result = Assembler::VC; break;
2594 default:
2595 ShouldNotReachHere();
2596 return Assembler::Condition(-1);
2597 }
2598
2599 // Check conversion
2600 if (cond & BoolTest::unsigned_compare) {
2601 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2602 } else {
2603 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2604 }
2605
2606 return result;
2607 }
2608
2609 // Binary src (Replicate con)
2610 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2611 if (n == nullptr || m == nullptr) {
2612 return false;
2613 }
2614
2615 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2616 return false;
2617 }
2618
2619 Node* imm_node = m->in(1);
2620 if (!imm_node->is_Con()) {
2621 return false;
2622 }
2623
2624 const Type* t = imm_node->bottom_type();
2625 if (!(t->isa_int() || t->isa_long())) {
2626 return false;
2627 }
2628
2629 switch (n->Opcode()) {
2630 case Op_AndV:
2631 case Op_OrV:
2632 case Op_XorV: {
2633 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2634 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2635 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2636 }
2637 case Op_AddVB:
2638 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2639 case Op_AddVS:
2640 case Op_AddVI:
2641 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2642 case Op_AddVL:
2643 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2644 default:
2645 return false;
2646 }
2647 }
2648
2649 // (XorV src (Replicate m1))
2650 // (XorVMask src (MaskAll m1))
2651 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2652 if (n != nullptr && m != nullptr) {
2653 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2654 VectorNode::is_all_ones_vector(m);
2655 }
2656 return false;
2657 }
2658
2659 // Should the matcher clone input 'm' of node 'n'?
2660 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2661 if (is_vshift_con_pattern(n, m) ||
2662 is_vector_bitwise_not_pattern(n, m) ||
2663 is_valid_sve_arith_imm_pattern(n, m) ||
2664 is_encode_and_store_pattern(n, m)) {
2665 mstack.push(m, Visit);
2666 return true;
2667 }
2668 return false;
2669 }
2670
2671 // Should the Matcher clone shifts on addressing modes, expecting them
2672 // to be subsumed into complex addressing expressions or compute them
2673 // into registers?
2674 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2675
2676 // Loads and stores with indirect memory input (e.g., volatile loads and
2677 // stores) do not subsume the input into complex addressing expressions. If
2678 // the addressing expression is input to at least one such load or store, do
2679 // not clone the addressing expression. Query needs_acquiring_load and
2680 // needs_releasing_store as a proxy for indirect memory input, as it is not
2681 // possible to directly query for indirect memory input at this stage.
2682 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2683 Node* n = m->fast_out(i);
2684 if (n->is_Load() && needs_acquiring_load(n)) {
2685 return false;
2686 }
2687 if (n->is_Store() && needs_releasing_store(n)) {
2688 return false;
2689 }
2690 }
2691
2692 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2693 return true;
2694 }
2695
2696 Node *off = m->in(AddPNode::Offset);
2697 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2698 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2699 // Are there other uses besides address expressions?
2700 !is_visited(off)) {
2701 address_visited.set(off->_idx); // Flag as address_visited
2702 mstack.push(off->in(2), Visit);
2703 Node *conv = off->in(1);
2704 if (conv->Opcode() == Op_ConvI2L &&
2705 // Are there other uses besides address expressions?
2706 !is_visited(conv)) {
2707 address_visited.set(conv->_idx); // Flag as address_visited
2708 mstack.push(conv->in(1), Pre_Visit);
2709 } else {
2710 mstack.push(conv, Pre_Visit);
2711 }
2712 address_visited.test_set(m->_idx); // Flag as address_visited
2713 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2714 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2715 return true;
2716 } else if (off->Opcode() == Op_ConvI2L &&
2717 // Are there other uses besides address expressions?
2718 !is_visited(off)) {
2719 address_visited.test_set(m->_idx); // Flag as address_visited
2720 address_visited.set(off->_idx); // Flag as address_visited
2721 mstack.push(off->in(1), Pre_Visit);
2722 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2723 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2724 return true;
2725 }
2726 return false;
2727 }
2728
2729 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2730 { \
2731 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2732 guarantee(DISP == 0, "mode not permitted for volatile"); \
2733 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2734 __ INSN(REG, as_Register(BASE)); \
2735 }
2736
2737
2738 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2739 {
2740 Address::extend scale;
2741
2742 // Hooboy, this is fugly. We need a way to communicate to the
2743 // encoder that the index needs to be sign extended, so we have to
2744 // enumerate all the cases.
2745 switch (opcode) {
2746 case INDINDEXSCALEDI2L:
2747 case INDINDEXSCALEDI2LN:
2748 case INDINDEXI2L:
2749 case INDINDEXI2LN:
2750 scale = Address::sxtw(size);
2751 break;
2752 default:
2753 scale = Address::lsl(size);
2754 }
2755
2756 if (index == -1) {
2757 return Address(base, disp);
2758 } else {
2759 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2760 return Address(base, as_Register(index), scale);
2761 }
2762 }
2763
2764
2765 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2766 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2767 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2768 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2769 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2770
2771 // Used for all non-volatile memory accesses. The use of
2772 // $mem->opcode() to discover whether this pattern uses sign-extended
2773 // offsets is something of a kludge.
2774 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2775 Register reg, int opcode,
2776 Register base, int index, int scale, int disp,
2777 int size_in_memory)
2778 {
2779 Address addr = mem2address(opcode, base, index, scale, disp);
2780 if (addr.getMode() == Address::base_plus_offset) {
2781 /* Fix up any out-of-range offsets. */
2782 assert_different_registers(rscratch1, base);
2783 assert_different_registers(rscratch1, reg);
2784 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2785 }
2786 (masm->*insn)(reg, addr);
2787 }
2788
2789 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2790 FloatRegister reg, int opcode,
2791 Register base, int index, int size, int disp,
2792 int size_in_memory)
2793 {
2794 Address::extend scale;
2795
2796 switch (opcode) {
2797 case INDINDEXSCALEDI2L:
2798 case INDINDEXSCALEDI2LN:
2799 scale = Address::sxtw(size);
2800 break;
2801 default:
2802 scale = Address::lsl(size);
2803 }
2804
2805 if (index == -1) {
2806 // Fix up any out-of-range offsets.
2807 assert_different_registers(rscratch1, base);
2808 Address addr = Address(base, disp);
2809 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2810 (masm->*insn)(reg, addr);
2811 } else {
2812 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2813 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2814 }
2815 }
2816
2817 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2818 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2819 int opcode, Register base, int index, int size, int disp)
2820 {
2821 if (index == -1) {
2822 (masm->*insn)(reg, T, Address(base, disp));
2823 } else {
2824 assert(disp == 0, "unsupported address mode");
2825 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2826 }
2827 }
2828
2829 %}
2830
2831
2832
2833 //----------ENCODING BLOCK-----------------------------------------------------
2834 // This block specifies the encoding classes used by the compiler to
2835 // output byte streams. Encoding classes are parameterized macros
2836 // used by Machine Instruction Nodes in order to generate the bit
2837 // encoding of the instruction. Operands specify their base encoding
2838 // interface with the interface keyword. There are currently
2839 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2840 // COND_INTER. REG_INTER causes an operand to generate a function
2841 // which returns its register number when queried. CONST_INTER causes
2842 // an operand to generate a function which returns the value of the
2843 // constant when queried. MEMORY_INTER causes an operand to generate
2844 // four functions which return the Base Register, the Index Register,
2845 // the Scale Value, and the Offset Value of the operand when queried.
2846 // COND_INTER causes an operand to generate six functions which return
2847 // the encoding code (ie - encoding bits for the instruction)
2848 // associated with each basic boolean condition for a conditional
2849 // instruction.
2850 //
2851 // Instructions specify two basic values for encoding. Again, a
2852 // function is available to check if the constant displacement is an
2853 // oop. They use the ins_encode keyword to specify their encoding
2854 // classes (which must be a sequence of enc_class names, and their
2855 // parameters, specified in the encoding block), and they use the
2856 // opcode keyword to specify, in order, their primary, secondary, and
2857 // tertiary opcode. Only the opcode sections which a particular
2858 // instruction needs for encoding need to be specified.
2859 encode %{
2860 // Build emit functions for each basic byte or larger field in the
2861 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2862 // from C++ code in the enc_class source block. Emit functions will
2863 // live in the main source block for now. In future, we can
2864 // generalize this by adding a syntax that specifies the sizes of
2865 // fields in an order, so that the adlc can build the emit functions
2866 // automagically
2867
2868 // catch all for unimplemented encodings
2869 enc_class enc_unimplemented %{
2870 __ unimplemented("C2 catch all");
2871 %}
2872
2873 // BEGIN Non-volatile memory access
2874
2875 // This encoding class is generated automatically from ad_encode.m4.
2876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2877 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2878 Register dst_reg = as_Register($dst$$reg);
2879 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2880 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2881 %}
2882
2883 // This encoding class is generated automatically from ad_encode.m4.
2884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2885 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2886 Register dst_reg = as_Register($dst$$reg);
2887 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2888 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2889 %}
2890
2891 // This encoding class is generated automatically from ad_encode.m4.
2892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2893 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2894 Register dst_reg = as_Register($dst$$reg);
2895 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2896 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2897 %}
2898
2899 // This encoding class is generated automatically from ad_encode.m4.
2900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2901 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2902 Register dst_reg = as_Register($dst$$reg);
2903 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2904 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2905 %}
2906
2907 // This encoding class is generated automatically from ad_encode.m4.
2908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2909 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2910 Register dst_reg = as_Register($dst$$reg);
2911 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2912 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2913 %}
2914
2915 // This encoding class is generated automatically from ad_encode.m4.
2916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2917 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2918 Register dst_reg = as_Register($dst$$reg);
2919 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2921 %}
2922
2923 // This encoding class is generated automatically from ad_encode.m4.
2924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2925 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2926 Register dst_reg = as_Register($dst$$reg);
2927 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2928 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2929 %}
2930
2931 // This encoding class is generated automatically from ad_encode.m4.
2932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2933 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2934 Register dst_reg = as_Register($dst$$reg);
2935 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2937 %}
2938
2939 // This encoding class is generated automatically from ad_encode.m4.
2940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2941 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2942 Register dst_reg = as_Register($dst$$reg);
2943 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2950 Register dst_reg = as_Register($dst$$reg);
2951 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2952 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2953 %}
2954
2955 // This encoding class is generated automatically from ad_encode.m4.
2956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2957 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2958 Register dst_reg = as_Register($dst$$reg);
2959 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2960 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2961 %}
2962
2963 // This encoding class is generated automatically from ad_encode.m4.
2964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2965 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2966 Register dst_reg = as_Register($dst$$reg);
2967 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2968 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2969 %}
2970
2971 // This encoding class is generated automatically from ad_encode.m4.
2972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2973 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2974 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2975 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2976 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2977 %}
2978
2979 // This encoding class is generated automatically from ad_encode.m4.
2980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2981 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2982 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2983 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2984 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2985 %}
2986
2987 // This encoding class is generated automatically from ad_encode.m4.
2988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2989 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2990 Register src_reg = as_Register($src$$reg);
2991 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2992 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2993 %}
2994
2995 // This encoding class is generated automatically from ad_encode.m4.
2996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2997 enc_class aarch64_enc_strb0(memory1 mem) %{
2998 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2999 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3000 %}
3001
3002 // This encoding class is generated automatically from ad_encode.m4.
3003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3004 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3005 Register src_reg = as_Register($src$$reg);
3006 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3007 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3008 %}
3009
3010 // This encoding class is generated automatically from ad_encode.m4.
3011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3012 enc_class aarch64_enc_strh0(memory2 mem) %{
3013 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3014 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3015 %}
3016
3017 // This encoding class is generated automatically from ad_encode.m4.
3018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3019 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3020 Register src_reg = as_Register($src$$reg);
3021 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3022 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3023 %}
3024
3025 // This encoding class is generated automatically from ad_encode.m4.
3026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3027 enc_class aarch64_enc_strw0(memory4 mem) %{
3028 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3029 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3030 %}
3031
3032 // This encoding class is generated automatically from ad_encode.m4.
3033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3034 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3035 Register src_reg = as_Register($src$$reg);
3036 // we sometimes get asked to store the stack pointer into the
3037 // current thread -- we cannot do that directly on AArch64
3038 if (src_reg == r31_sp) {
3039 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3040 __ mov(rscratch2, sp);
3041 src_reg = rscratch2;
3042 }
3043 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3044 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3045 %}
3046
3047 // This encoding class is generated automatically from ad_encode.m4.
3048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3049 enc_class aarch64_enc_str0(memory8 mem) %{
3050 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3051 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3052 %}
3053
3054 // This encoding class is generated automatically from ad_encode.m4.
3055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3056 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3057 FloatRegister src_reg = as_FloatRegister($src$$reg);
3058 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3059 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3060 %}
3061
3062 // This encoding class is generated automatically from ad_encode.m4.
3063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3064 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3065 FloatRegister src_reg = as_FloatRegister($src$$reg);
3066 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3067 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3068 %}
3069
3070 // This encoding class is generated automatically from ad_encode.m4.
3071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3072 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3073 __ membar(Assembler::StoreStore);
3074 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3075 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3076 %}
3077
3078 // END Non-volatile memory access
3079
3080 // Vector loads and stores
3081 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3082 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3083 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3084 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3085 %}
3086
3087 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3088 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3089 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3090 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3091 %}
3092
3093 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3094 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3095 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3096 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3097 %}
3098
3099 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3100 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3101 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3102 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3103 %}
3104
3105 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3106 FloatRegister src_reg = as_FloatRegister($src$$reg);
3107 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3108 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3109 %}
3110
3111 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3112 FloatRegister src_reg = as_FloatRegister($src$$reg);
3113 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3114 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3115 %}
3116
3117 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3118 FloatRegister src_reg = as_FloatRegister($src$$reg);
3119 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3120 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3121 %}
3122
3123 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3124 FloatRegister src_reg = as_FloatRegister($src$$reg);
3125 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3126 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3127 %}
3128
3129 // volatile loads and stores
3130
3131 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3132 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3133 rscratch1, stlrb);
3134 %}
3135
3136 enc_class aarch64_enc_stlrb0(memory mem) %{
3137 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3138 rscratch1, stlrb);
3139 %}
3140
3141 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3142 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3143 rscratch1, stlrh);
3144 %}
3145
3146 enc_class aarch64_enc_stlrh0(memory mem) %{
3147 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3148 rscratch1, stlrh);
3149 %}
3150
3151 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3152 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3153 rscratch1, stlrw);
3154 %}
3155
3156 enc_class aarch64_enc_stlrw0(memory mem) %{
3157 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3158 rscratch1, stlrw);
3159 %}
3160
3161 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3162 Register dst_reg = as_Register($dst$$reg);
3163 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3164 rscratch1, ldarb);
3165 __ sxtbw(dst_reg, dst_reg);
3166 %}
3167
3168 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3169 Register dst_reg = as_Register($dst$$reg);
3170 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3171 rscratch1, ldarb);
3172 __ sxtb(dst_reg, dst_reg);
3173 %}
3174
3175 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3176 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3177 rscratch1, ldarb);
3178 %}
3179
3180 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3181 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3182 rscratch1, ldarb);
3183 %}
3184
3185 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3186 Register dst_reg = as_Register($dst$$reg);
3187 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3188 rscratch1, ldarh);
3189 __ sxthw(dst_reg, dst_reg);
3190 %}
3191
3192 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3193 Register dst_reg = as_Register($dst$$reg);
3194 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3195 rscratch1, ldarh);
3196 __ sxth(dst_reg, dst_reg);
3197 %}
3198
3199 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3200 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3201 rscratch1, ldarh);
3202 %}
3203
3204 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3205 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3206 rscratch1, ldarh);
3207 %}
3208
3209 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3210 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3211 rscratch1, ldarw);
3212 %}
3213
3214 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3215 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3216 rscratch1, ldarw);
3217 %}
3218
3219 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3220 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3221 rscratch1, ldar);
3222 %}
3223
3224 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3225 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3226 rscratch1, ldarw);
3227 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3228 %}
3229
3230 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3231 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3232 rscratch1, ldar);
3233 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3234 %}
3235
3236 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3237 Register src_reg = as_Register($src$$reg);
3238 // we sometimes get asked to store the stack pointer into the
3239 // current thread -- we cannot do that directly on AArch64
3240 if (src_reg == r31_sp) {
3241 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3242 __ mov(rscratch2, sp);
3243 src_reg = rscratch2;
3244 }
3245 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3246 rscratch1, stlr);
3247 %}
3248
3249 enc_class aarch64_enc_stlr0(memory mem) %{
3250 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3251 rscratch1, stlr);
3252 %}
3253
3254 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3255 {
3256 FloatRegister src_reg = as_FloatRegister($src$$reg);
3257 __ fmovs(rscratch2, src_reg);
3258 }
3259 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3260 rscratch1, stlrw);
3261 %}
3262
3263 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3264 {
3265 FloatRegister src_reg = as_FloatRegister($src$$reg);
3266 __ fmovd(rscratch2, src_reg);
3267 }
3268 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3269 rscratch1, stlr);
3270 %}
3271
3272 // synchronized read/update encodings
3273
3274 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3275 Register dst_reg = as_Register($dst$$reg);
3276 Register base = as_Register($mem$$base);
3277 int index = $mem$$index;
3278 int scale = $mem$$scale;
3279 int disp = $mem$$disp;
3280 if (index == -1) {
3281 if (disp != 0) {
3282 __ lea(rscratch1, Address(base, disp));
3283 __ ldaxr(dst_reg, rscratch1);
3284 } else {
3285 // TODO
3286 // should we ever get anything other than this case?
3287 __ ldaxr(dst_reg, base);
3288 }
3289 } else {
3290 Register index_reg = as_Register(index);
3291 if (disp == 0) {
3292 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3293 __ ldaxr(dst_reg, rscratch1);
3294 } else {
3295 __ lea(rscratch1, Address(base, disp));
3296 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3297 __ ldaxr(dst_reg, rscratch1);
3298 }
3299 }
3300 %}
3301
3302 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3303 Register src_reg = as_Register($src$$reg);
3304 Register base = as_Register($mem$$base);
3305 int index = $mem$$index;
3306 int scale = $mem$$scale;
3307 int disp = $mem$$disp;
3308 if (index == -1) {
3309 if (disp != 0) {
3310 __ lea(rscratch2, Address(base, disp));
3311 __ stlxr(rscratch1, src_reg, rscratch2);
3312 } else {
3313 // TODO
3314 // should we ever get anything other than this case?
3315 __ stlxr(rscratch1, src_reg, base);
3316 }
3317 } else {
3318 Register index_reg = as_Register(index);
3319 if (disp == 0) {
3320 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3321 __ stlxr(rscratch1, src_reg, rscratch2);
3322 } else {
3323 __ lea(rscratch2, Address(base, disp));
3324 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3325 __ stlxr(rscratch1, src_reg, rscratch2);
3326 }
3327 }
3328 __ cmpw(rscratch1, zr);
3329 %}
3330
3331 // prefetch encodings
3332
3333 enc_class aarch64_enc_prefetchw(memory mem) %{
3334 Register base = as_Register($mem$$base);
3335 int index = $mem$$index;
3336 int scale = $mem$$scale;
3337 int disp = $mem$$disp;
3338 if (index == -1) {
3339 // Fix up any out-of-range offsets.
3340 assert_different_registers(rscratch1, base);
3341 Address addr = Address(base, disp);
3342 addr = __ legitimize_address(addr, 8, rscratch1);
3343 __ prfm(addr, PSTL1KEEP);
3344 } else {
3345 Register index_reg = as_Register(index);
3346 if (disp == 0) {
3347 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3348 } else {
3349 __ lea(rscratch1, Address(base, disp));
3350 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3351 }
3352 }
3353 %}
3354
3355 // mov encodings
3356
3357 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3358 uint32_t con = (uint32_t)$src$$constant;
3359 Register dst_reg = as_Register($dst$$reg);
3360 if (con == 0) {
3361 __ movw(dst_reg, zr);
3362 } else {
3363 __ movw(dst_reg, con);
3364 }
3365 %}
3366
3367 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3368 Register dst_reg = as_Register($dst$$reg);
3369 uint64_t con = (uint64_t)$src$$constant;
3370 if (con == 0) {
3371 __ mov(dst_reg, zr);
3372 } else {
3373 __ mov(dst_reg, con);
3374 }
3375 %}
3376
3377 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3378 Register dst_reg = as_Register($dst$$reg);
3379 address con = (address)$src$$constant;
3380 if (con == nullptr || con == (address)1) {
3381 ShouldNotReachHere();
3382 } else {
3383 relocInfo::relocType rtype = $src->constant_reloc();
3384 if (rtype == relocInfo::oop_type) {
3385 __ movoop(dst_reg, (jobject)con);
3386 } else if (rtype == relocInfo::metadata_type) {
3387 __ mov_metadata(dst_reg, (Metadata*)con);
3388 } else {
3389 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3390 // load fake address constants using a normal move
3391 if (! __ is_valid_AArch64_address(con) ||
3392 con < (address)(uintptr_t)os::vm_page_size() ||
3393 rtype == relocInfo::none) {
3394 __ mov(dst_reg, con);
3395 } else {
3396 // use shorter adrp/add sequence for external_word relocation
3397 uint64_t offset;
3398 __ adrp(dst_reg, Address(con, rtype), offset);
3399 __ add(dst_reg, dst_reg, offset);
3400 }
3401 }
3402 }
3403 %}
3404
3405 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3406 Register dst_reg = as_Register($dst$$reg);
3407 __ mov(dst_reg, zr);
3408 %}
3409
3410 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3411 Register dst_reg = as_Register($dst$$reg);
3412 __ mov(dst_reg, (uint64_t)1);
3413 %}
3414
3415 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3416 Register dst_reg = as_Register($dst$$reg);
3417 address con = (address)$src$$constant;
3418 if (con == nullptr) {
3419 ShouldNotReachHere();
3420 } else {
3421 relocInfo::relocType rtype = $src->constant_reloc();
3422 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3423 __ set_narrow_oop(dst_reg, (jobject)con);
3424 }
3425 %}
3426
3427 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3428 Register dst_reg = as_Register($dst$$reg);
3429 __ mov(dst_reg, zr);
3430 %}
3431
3432 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3433 Register dst_reg = as_Register($dst$$reg);
3434 address con = (address)$src$$constant;
3435 if (con == nullptr) {
3436 ShouldNotReachHere();
3437 } else {
3438 relocInfo::relocType rtype = $src->constant_reloc();
3439 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3440 __ set_narrow_klass(dst_reg, (Klass *)con);
3441 }
3442 %}
3443
3444 // arithmetic encodings
3445
3446 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3447 Register dst_reg = as_Register($dst$$reg);
3448 Register src_reg = as_Register($src1$$reg);
3449 int32_t con = (int32_t)$src2$$constant;
3450 // add has primary == 0, subtract has primary == 1
3451 if ($primary) { con = -con; }
3452 if (con < 0) {
3453 __ subw(dst_reg, src_reg, -con);
3454 } else {
3455 __ addw(dst_reg, src_reg, con);
3456 }
3457 %}
3458
3459 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3460 Register dst_reg = as_Register($dst$$reg);
3461 Register src_reg = as_Register($src1$$reg);
3462 int32_t con = (int32_t)$src2$$constant;
3463 // add has primary == 0, subtract has primary == 1
3464 if ($primary) { con = -con; }
3465 if (con < 0) {
3466 __ sub(dst_reg, src_reg, -con);
3467 } else {
3468 __ add(dst_reg, src_reg, con);
3469 }
3470 %}
3471
3472 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3473 Register dst_reg = as_Register($dst$$reg);
3474 Register src1_reg = as_Register($src1$$reg);
3475 Register src2_reg = as_Register($src2$$reg);
3476 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3477 %}
3478
3479 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3480 Register dst_reg = as_Register($dst$$reg);
3481 Register src1_reg = as_Register($src1$$reg);
3482 Register src2_reg = as_Register($src2$$reg);
3483 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3484 %}
3485
3486 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3487 Register dst_reg = as_Register($dst$$reg);
3488 Register src1_reg = as_Register($src1$$reg);
3489 Register src2_reg = as_Register($src2$$reg);
3490 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3491 %}
3492
3493 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3494 Register dst_reg = as_Register($dst$$reg);
3495 Register src1_reg = as_Register($src1$$reg);
3496 Register src2_reg = as_Register($src2$$reg);
3497 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3498 %}
3499
3500 // compare instruction encodings
3501
3502 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3503 Register reg1 = as_Register($src1$$reg);
3504 Register reg2 = as_Register($src2$$reg);
3505 __ cmpw(reg1, reg2);
3506 %}
3507
3508 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3509 Register reg = as_Register($src1$$reg);
3510 int32_t val = $src2$$constant;
3511 if (val >= 0) {
3512 __ subsw(zr, reg, val);
3513 } else {
3514 __ addsw(zr, reg, -val);
3515 }
3516 %}
3517
3518 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3519 Register reg1 = as_Register($src1$$reg);
3520 uint32_t val = (uint32_t)$src2$$constant;
3521 __ movw(rscratch1, val);
3522 __ cmpw(reg1, rscratch1);
3523 %}
3524
3525 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3526 Register reg1 = as_Register($src1$$reg);
3527 Register reg2 = as_Register($src2$$reg);
3528 __ cmp(reg1, reg2);
3529 %}
3530
3531 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3532 Register reg = as_Register($src1$$reg);
3533 int64_t val = $src2$$constant;
3534 if (val >= 0) {
3535 __ subs(zr, reg, val);
3536 } else if (val != -val) {
3537 __ adds(zr, reg, -val);
3538 } else {
3539 // aargh, Long.MIN_VALUE is a special case
3540 __ orr(rscratch1, zr, (uint64_t)val);
3541 __ subs(zr, reg, rscratch1);
3542 }
3543 %}
3544
3545 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3546 Register reg1 = as_Register($src1$$reg);
3547 uint64_t val = (uint64_t)$src2$$constant;
3548 __ mov(rscratch1, val);
3549 __ cmp(reg1, rscratch1);
3550 %}
3551
3552 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3553 Register reg1 = as_Register($src1$$reg);
3554 Register reg2 = as_Register($src2$$reg);
3555 __ cmp(reg1, reg2);
3556 %}
3557
3558 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3559 Register reg1 = as_Register($src1$$reg);
3560 Register reg2 = as_Register($src2$$reg);
3561 __ cmpw(reg1, reg2);
3562 %}
3563
3564 enc_class aarch64_enc_testp(iRegP src) %{
3565 Register reg = as_Register($src$$reg);
3566 __ cmp(reg, zr);
3567 %}
3568
3569 enc_class aarch64_enc_testn(iRegN src) %{
3570 Register reg = as_Register($src$$reg);
3571 __ cmpw(reg, zr);
3572 %}
3573
3574 enc_class aarch64_enc_b(label lbl) %{
3575 Label *L = $lbl$$label;
3576 __ b(*L);
3577 %}
3578
3579 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3580 Label *L = $lbl$$label;
3581 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3582 %}
3583
3584 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3585 Label *L = $lbl$$label;
3586 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3587 %}
3588
3589 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3590 %{
3591 Register sub_reg = as_Register($sub$$reg);
3592 Register super_reg = as_Register($super$$reg);
3593 Register temp_reg = as_Register($temp$$reg);
3594 Register result_reg = as_Register($result$$reg);
3595
3596 Label miss;
3597 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3598 nullptr, &miss,
3599 /*set_cond_codes:*/ true);
3600 if ($primary) {
3601 __ mov(result_reg, zr);
3602 }
3603 __ bind(miss);
3604 %}
3605
3606 enc_class aarch64_enc_java_static_call(method meth) %{
3607 address addr = (address)$meth$$method;
3608 address call;
3609 if (!_method) {
3610 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3611 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3612 if (call == nullptr) {
3613 ciEnv::current()->record_failure("CodeCache is full");
3614 return;
3615 }
3616 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3617 // The NOP here is purely to ensure that eliding a call to
3618 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3619 __ nop();
3620 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3621 } else {
3622 int method_index = resolved_method_index(masm);
3623 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3624 : static_call_Relocation::spec(method_index);
3625 call = __ trampoline_call(Address(addr, rspec));
3626 if (call == nullptr) {
3627 ciEnv::current()->record_failure("CodeCache is full");
3628 return;
3629 }
3630 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3631 // Calls of the same statically bound method can share
3632 // a stub to the interpreter.
3633 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3634 } else {
3635 // Emit stub for static call
3636 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3637 if (stub == nullptr) {
3638 ciEnv::current()->record_failure("CodeCache is full");
3639 return;
3640 }
3641 }
3642 }
3643
3644 __ post_call_nop();
3645
3646 // Only non uncommon_trap calls need to reinitialize ptrue.
3647 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3648 __ reinitialize_ptrue();
3649 }
3650 %}
3651
3652 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3653 int method_index = resolved_method_index(masm);
3654 address call = __ ic_call((address)$meth$$method, method_index);
3655 if (call == nullptr) {
3656 ciEnv::current()->record_failure("CodeCache is full");
3657 return;
3658 }
3659 __ post_call_nop();
3660 if (Compile::current()->max_vector_size() > 0) {
3661 __ reinitialize_ptrue();
3662 }
3663 %}
3664
3665 enc_class aarch64_enc_call_epilog() %{
3666 if (VerifyStackAtCalls) {
3667 // Check that stack depth is unchanged: find majik cookie on stack
3668 __ call_Unimplemented();
3669 }
3670 %}
3671
3672 enc_class aarch64_enc_java_to_runtime(method meth) %{
3673 // some calls to generated routines (arraycopy code) are scheduled
3674 // by C2 as runtime calls. if so we can call them using a br (they
3675 // will be in a reachable segment) otherwise we have to use a blr
3676 // which loads the absolute address into a register.
3677 address entry = (address)$meth$$method;
3678 CodeBlob *cb = CodeCache::find_blob(entry);
3679 if (cb) {
3680 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3681 if (call == nullptr) {
3682 ciEnv::current()->record_failure("CodeCache is full");
3683 return;
3684 }
3685 __ post_call_nop();
3686 } else {
3687 Label retaddr;
3688 // Make the anchor frame walkable
3689 __ adr(rscratch2, retaddr);
3690 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3691 __ lea(rscratch1, RuntimeAddress(entry));
3692 __ blr(rscratch1);
3693 __ bind(retaddr);
3694 __ post_call_nop();
3695 }
3696 if (Compile::current()->max_vector_size() > 0) {
3697 __ reinitialize_ptrue();
3698 }
3699 %}
3700
3701 enc_class aarch64_enc_rethrow() %{
3702 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3703 %}
3704
3705 enc_class aarch64_enc_ret() %{
3706 #ifdef ASSERT
3707 if (Compile::current()->max_vector_size() > 0) {
3708 __ verify_ptrue();
3709 }
3710 #endif
3711 __ ret(lr);
3712 %}
3713
3714 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3715 Register target_reg = as_Register($jump_target$$reg);
3716 __ br(target_reg);
3717 %}
3718
3719 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3720 Register target_reg = as_Register($jump_target$$reg);
3721 // exception oop should be in r0
3722 // ret addr has been popped into lr
3723 // callee expects it in r3
3724 __ mov(r3, lr);
3725 __ br(target_reg);
3726 %}
3727
3728 %}
3729
3730 //----------FRAME--------------------------------------------------------------
3731 // Definition of frame structure and management information.
3732 //
3733 // S T A C K L A Y O U T Allocators stack-slot number
3734 // | (to get allocators register number
3735 // G Owned by | | v add OptoReg::stack0())
3736 // r CALLER | |
3737 // o | +--------+ pad to even-align allocators stack-slot
3738 // w V | pad0 | numbers; owned by CALLER
3739 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3740 // h ^ | in | 5
3741 // | | args | 4 Holes in incoming args owned by SELF
3742 // | | | | 3
3743 // | | +--------+
3744 // V | | old out| Empty on Intel, window on Sparc
3745 // | old |preserve| Must be even aligned.
3746 // | SP-+--------+----> Matcher::_old_SP, even aligned
3747 // | | in | 3 area for Intel ret address
3748 // Owned by |preserve| Empty on Sparc.
3749 // SELF +--------+
3750 // | | pad2 | 2 pad to align old SP
3751 // | +--------+ 1
3752 // | | locks | 0
3753 // | +--------+----> OptoReg::stack0(), even aligned
3754 // | | pad1 | 11 pad to align new SP
3755 // | +--------+
3756 // | | | 10
3757 // | | spills | 9 spills
3758 // V | | 8 (pad0 slot for callee)
3759 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3760 // ^ | out | 7
3761 // | | args | 6 Holes in outgoing args owned by CALLEE
3762 // Owned by +--------+
3763 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3764 // | new |preserve| Must be even-aligned.
3765 // | SP-+--------+----> Matcher::_new_SP, even aligned
3766 // | | |
3767 //
3768 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3769 // known from SELF's arguments and the Java calling convention.
3770 // Region 6-7 is determined per call site.
3771 // Note 2: If the calling convention leaves holes in the incoming argument
3772 // area, those holes are owned by SELF. Holes in the outgoing area
3773 // are owned by the CALLEE. Holes should not be necessary in the
3774 // incoming area, as the Java calling convention is completely under
3775 // the control of the AD file. Doubles can be sorted and packed to
3776 // avoid holes. Holes in the outgoing arguments may be necessary for
3777 // varargs C calling conventions.
3778 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3779 // even aligned with pad0 as needed.
3780 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3781 // (the latter is true on Intel but is it false on AArch64?)
3782 // region 6-11 is even aligned; it may be padded out more so that
3783 // the region from SP to FP meets the minimum stack alignment.
3784 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3785 // alignment. Region 11, pad1, may be dynamically extended so that
3786 // SP meets the minimum alignment.
3787
3788 frame %{
3789 // These three registers define part of the calling convention
3790 // between compiled code and the interpreter.
3791
3792 // Inline Cache Register or Method for I2C.
3793 inline_cache_reg(R12);
3794
3795 // Number of stack slots consumed by locking an object
3796 sync_stack_slots(2);
3797
3798 // Compiled code's Frame Pointer
3799 frame_pointer(R31);
3800
3801 // Stack alignment requirement
3802 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3803
3804 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3805 // for calls to C. Supports the var-args backing area for register parms.
3806 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3807
3808 // The after-PROLOG location of the return address. Location of
3809 // return address specifies a type (REG or STACK) and a number
3810 // representing the register number (i.e. - use a register name) or
3811 // stack slot.
3812 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3813 // Otherwise, it is above the locks and verification slot and alignment word
3814 // TODO this may well be correct but need to check why that - 2 is there
3815 // ppc port uses 0 but we definitely need to allow for fixed_slots
3816 // which folds in the space used for monitors
3817 return_addr(STACK - 2 +
3818 align_up((Compile::current()->in_preserve_stack_slots() +
3819 Compile::current()->fixed_slots()),
3820 stack_alignment_in_slots()));
3821
3822 // Location of compiled Java return values. Same as C for now.
3823 return_value
3824 %{
3825 // TODO do we allow ideal_reg == Op_RegN???
3826 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3827 "only return normal values");
3828
3829 static const int lo[Op_RegL + 1] = { // enum name
3830 0, // Op_Node
3831 0, // Op_Set
3832 R0_num, // Op_RegN
3833 R0_num, // Op_RegI
3834 R0_num, // Op_RegP
3835 V0_num, // Op_RegF
3836 V0_num, // Op_RegD
3837 R0_num // Op_RegL
3838 };
3839
3840 static const int hi[Op_RegL + 1] = { // enum name
3841 0, // Op_Node
3842 0, // Op_Set
3843 OptoReg::Bad, // Op_RegN
3844 OptoReg::Bad, // Op_RegI
3845 R0_H_num, // Op_RegP
3846 OptoReg::Bad, // Op_RegF
3847 V0_H_num, // Op_RegD
3848 R0_H_num // Op_RegL
3849 };
3850
3851 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3852 %}
3853 %}
3854
3855 //----------ATTRIBUTES---------------------------------------------------------
3856 //----------Operand Attributes-------------------------------------------------
3857 op_attrib op_cost(1); // Required cost attribute
3858
3859 //----------Instruction Attributes---------------------------------------------
3860 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3861 ins_attrib ins_size(32); // Required size attribute (in bits)
3862 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3863 // a non-matching short branch variant
3864 // of some long branch?
3865 ins_attrib ins_alignment(4); // Required alignment attribute (must
3866 // be a power of 2) specifies the
3867 // alignment that some part of the
3868 // instruction (not necessarily the
3869 // start) requires. If > 1, a
3870 // compute_padding() function must be
3871 // provided for the instruction
3872
3873 // Whether this node is expanded during code emission into a sequence of
3874 // instructions and the first instruction can perform an implicit null check.
3875 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3876
3877 //----------OPERANDS-----------------------------------------------------------
3878 // Operand definitions must precede instruction definitions for correct parsing
3879 // in the ADLC because operands constitute user defined types which are used in
3880 // instruction definitions.
3881
3882 //----------Simple Operands----------------------------------------------------
3883
3884 // Integer operands 32 bit
3885 // 32 bit immediate
3886 operand immI()
3887 %{
3888 match(ConI);
3889
3890 op_cost(0);
3891 format %{ %}
3892 interface(CONST_INTER);
3893 %}
3894
3895 // 32 bit zero
3896 operand immI0()
3897 %{
3898 predicate(n->get_int() == 0);
3899 match(ConI);
3900
3901 op_cost(0);
3902 format %{ %}
3903 interface(CONST_INTER);
3904 %}
3905
3906 // 32 bit unit increment
3907 operand immI_1()
3908 %{
3909 predicate(n->get_int() == 1);
3910 match(ConI);
3911
3912 op_cost(0);
3913 format %{ %}
3914 interface(CONST_INTER);
3915 %}
3916
3917 // 32 bit unit decrement
3918 operand immI_M1()
3919 %{
3920 predicate(n->get_int() == -1);
3921 match(ConI);
3922
3923 op_cost(0);
3924 format %{ %}
3925 interface(CONST_INTER);
3926 %}
3927
3928 // Shift values for add/sub extension shift
3929 operand immIExt()
3930 %{
3931 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3932 match(ConI);
3933
3934 op_cost(0);
3935 format %{ %}
3936 interface(CONST_INTER);
3937 %}
3938
3939 operand immI_gt_1()
3940 %{
3941 predicate(n->get_int() > 1);
3942 match(ConI);
3943
3944 op_cost(0);
3945 format %{ %}
3946 interface(CONST_INTER);
3947 %}
3948
3949 operand immI_le_4()
3950 %{
3951 predicate(n->get_int() <= 4);
3952 match(ConI);
3953
3954 op_cost(0);
3955 format %{ %}
3956 interface(CONST_INTER);
3957 %}
3958
3959 operand immI_16()
3960 %{
3961 predicate(n->get_int() == 16);
3962 match(ConI);
3963
3964 op_cost(0);
3965 format %{ %}
3966 interface(CONST_INTER);
3967 %}
3968
3969 operand immI_24()
3970 %{
3971 predicate(n->get_int() == 24);
3972 match(ConI);
3973
3974 op_cost(0);
3975 format %{ %}
3976 interface(CONST_INTER);
3977 %}
3978
3979 operand immI_32()
3980 %{
3981 predicate(n->get_int() == 32);
3982 match(ConI);
3983
3984 op_cost(0);
3985 format %{ %}
3986 interface(CONST_INTER);
3987 %}
3988
3989 operand immI_48()
3990 %{
3991 predicate(n->get_int() == 48);
3992 match(ConI);
3993
3994 op_cost(0);
3995 format %{ %}
3996 interface(CONST_INTER);
3997 %}
3998
3999 operand immI_56()
4000 %{
4001 predicate(n->get_int() == 56);
4002 match(ConI);
4003
4004 op_cost(0);
4005 format %{ %}
4006 interface(CONST_INTER);
4007 %}
4008
4009 operand immI_255()
4010 %{
4011 predicate(n->get_int() == 255);
4012 match(ConI);
4013
4014 op_cost(0);
4015 format %{ %}
4016 interface(CONST_INTER);
4017 %}
4018
4019 operand immI_65535()
4020 %{
4021 predicate(n->get_int() == 65535);
4022 match(ConI);
4023
4024 op_cost(0);
4025 format %{ %}
4026 interface(CONST_INTER);
4027 %}
4028
4029 operand immI_positive()
4030 %{
4031 predicate(n->get_int() > 0);
4032 match(ConI);
4033
4034 op_cost(0);
4035 format %{ %}
4036 interface(CONST_INTER);
4037 %}
4038
4039 // BoolTest condition for signed compare
4040 operand immI_cmp_cond()
4041 %{
4042 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4043 match(ConI);
4044
4045 op_cost(0);
4046 format %{ %}
4047 interface(CONST_INTER);
4048 %}
4049
4050 // BoolTest condition for unsigned compare
4051 operand immI_cmpU_cond()
4052 %{
4053 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4054 match(ConI);
4055
4056 op_cost(0);
4057 format %{ %}
4058 interface(CONST_INTER);
4059 %}
4060
4061 operand immL_255()
4062 %{
4063 predicate(n->get_long() == 255L);
4064 match(ConL);
4065
4066 op_cost(0);
4067 format %{ %}
4068 interface(CONST_INTER);
4069 %}
4070
4071 operand immL_65535()
4072 %{
4073 predicate(n->get_long() == 65535L);
4074 match(ConL);
4075
4076 op_cost(0);
4077 format %{ %}
4078 interface(CONST_INTER);
4079 %}
4080
4081 operand immL_4294967295()
4082 %{
4083 predicate(n->get_long() == 4294967295L);
4084 match(ConL);
4085
4086 op_cost(0);
4087 format %{ %}
4088 interface(CONST_INTER);
4089 %}
4090
4091 operand immL_bitmask()
4092 %{
4093 predicate((n->get_long() != 0)
4094 && ((n->get_long() & 0xc000000000000000l) == 0)
4095 && is_power_of_2(n->get_long() + 1));
4096 match(ConL);
4097
4098 op_cost(0);
4099 format %{ %}
4100 interface(CONST_INTER);
4101 %}
4102
4103 operand immI_bitmask()
4104 %{
4105 predicate((n->get_int() != 0)
4106 && ((n->get_int() & 0xc0000000) == 0)
4107 && is_power_of_2(n->get_int() + 1));
4108 match(ConI);
4109
4110 op_cost(0);
4111 format %{ %}
4112 interface(CONST_INTER);
4113 %}
4114
4115 operand immL_positive_bitmaskI()
4116 %{
4117 predicate((n->get_long() != 0)
4118 && ((julong)n->get_long() < 0x80000000ULL)
4119 && is_power_of_2(n->get_long() + 1));
4120 match(ConL);
4121
4122 op_cost(0);
4123 format %{ %}
4124 interface(CONST_INTER);
4125 %}
4126
4127 // Scale values for scaled offset addressing modes (up to long but not quad)
4128 operand immIScale()
4129 %{
4130 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4131 match(ConI);
4132
4133 op_cost(0);
4134 format %{ %}
4135 interface(CONST_INTER);
4136 %}
4137
4138 // 5 bit signed integer
4139 operand immI5()
4140 %{
4141 predicate(Assembler::is_simm(n->get_int(), 5));
4142 match(ConI);
4143
4144 op_cost(0);
4145 format %{ %}
4146 interface(CONST_INTER);
4147 %}
4148
4149 // 7 bit unsigned integer
4150 operand immIU7()
4151 %{
4152 predicate(Assembler::is_uimm(n->get_int(), 7));
4153 match(ConI);
4154
4155 op_cost(0);
4156 format %{ %}
4157 interface(CONST_INTER);
4158 %}
4159
4160 // Offset for scaled or unscaled immediate loads and stores
4161 operand immIOffset()
4162 %{
4163 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4164 match(ConI);
4165
4166 op_cost(0);
4167 format %{ %}
4168 interface(CONST_INTER);
4169 %}
4170
4171 operand immIOffset1()
4172 %{
4173 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4174 match(ConI);
4175
4176 op_cost(0);
4177 format %{ %}
4178 interface(CONST_INTER);
4179 %}
4180
4181 operand immIOffset2()
4182 %{
4183 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4184 match(ConI);
4185
4186 op_cost(0);
4187 format %{ %}
4188 interface(CONST_INTER);
4189 %}
4190
4191 operand immIOffset4()
4192 %{
4193 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4194 match(ConI);
4195
4196 op_cost(0);
4197 format %{ %}
4198 interface(CONST_INTER);
4199 %}
4200
4201 operand immIOffset8()
4202 %{
4203 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4204 match(ConI);
4205
4206 op_cost(0);
4207 format %{ %}
4208 interface(CONST_INTER);
4209 %}
4210
4211 operand immIOffset16()
4212 %{
4213 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4214 match(ConI);
4215
4216 op_cost(0);
4217 format %{ %}
4218 interface(CONST_INTER);
4219 %}
4220
4221 operand immLOffset()
4222 %{
4223 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4224 match(ConL);
4225
4226 op_cost(0);
4227 format %{ %}
4228 interface(CONST_INTER);
4229 %}
4230
4231 operand immLoffset1()
4232 %{
4233 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4234 match(ConL);
4235
4236 op_cost(0);
4237 format %{ %}
4238 interface(CONST_INTER);
4239 %}
4240
4241 operand immLoffset2()
4242 %{
4243 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4244 match(ConL);
4245
4246 op_cost(0);
4247 format %{ %}
4248 interface(CONST_INTER);
4249 %}
4250
4251 operand immLoffset4()
4252 %{
4253 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4254 match(ConL);
4255
4256 op_cost(0);
4257 format %{ %}
4258 interface(CONST_INTER);
4259 %}
4260
4261 operand immLoffset8()
4262 %{
4263 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4264 match(ConL);
4265
4266 op_cost(0);
4267 format %{ %}
4268 interface(CONST_INTER);
4269 %}
4270
4271 operand immLoffset16()
4272 %{
4273 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4274 match(ConL);
4275
4276 op_cost(0);
4277 format %{ %}
4278 interface(CONST_INTER);
4279 %}
4280
4281 // 5 bit signed long integer
4282 operand immL5()
4283 %{
4284 predicate(Assembler::is_simm(n->get_long(), 5));
4285 match(ConL);
4286
4287 op_cost(0);
4288 format %{ %}
4289 interface(CONST_INTER);
4290 %}
4291
4292 // 7 bit unsigned long integer
4293 operand immLU7()
4294 %{
4295 predicate(Assembler::is_uimm(n->get_long(), 7));
4296 match(ConL);
4297
4298 op_cost(0);
4299 format %{ %}
4300 interface(CONST_INTER);
4301 %}
4302
4303 // 8 bit signed value.
4304 operand immI8()
4305 %{
4306 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4307 match(ConI);
4308
4309 op_cost(0);
4310 format %{ %}
4311 interface(CONST_INTER);
4312 %}
4313
4314 // 8 bit signed value (simm8), or #simm8 LSL 8.
4315 operand immIDupV()
4316 %{
4317 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4318 match(ConI);
4319
4320 op_cost(0);
4321 format %{ %}
4322 interface(CONST_INTER);
4323 %}
4324
4325 // 8 bit signed value (simm8), or #simm8 LSL 8.
4326 operand immLDupV()
4327 %{
4328 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4329 match(ConL);
4330
4331 op_cost(0);
4332 format %{ %}
4333 interface(CONST_INTER);
4334 %}
4335
4336 // 8 bit signed value (simm8), or #simm8 LSL 8.
4337 operand immHDupV()
4338 %{
4339 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4340 match(ConH);
4341
4342 op_cost(0);
4343 format %{ %}
4344 interface(CONST_INTER);
4345 %}
4346
4347 // 8 bit integer valid for vector add sub immediate
4348 operand immBAddSubV()
4349 %{
4350 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4351 match(ConI);
4352
4353 op_cost(0);
4354 format %{ %}
4355 interface(CONST_INTER);
4356 %}
4357
4358 // 32 bit integer valid for add sub immediate
4359 operand immIAddSub()
4360 %{
4361 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4362 match(ConI);
4363 op_cost(0);
4364 format %{ %}
4365 interface(CONST_INTER);
4366 %}
4367
4368 // 32 bit integer valid for vector add sub immediate
4369 operand immIAddSubV()
4370 %{
4371 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4372 match(ConI);
4373
4374 op_cost(0);
4375 format %{ %}
4376 interface(CONST_INTER);
4377 %}
4378
4379 // 32 bit unsigned integer valid for logical immediate
4380
4381 operand immBLog()
4382 %{
4383 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4384 match(ConI);
4385
4386 op_cost(0);
4387 format %{ %}
4388 interface(CONST_INTER);
4389 %}
4390
4391 operand immSLog()
4392 %{
4393 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4394 match(ConI);
4395
4396 op_cost(0);
4397 format %{ %}
4398 interface(CONST_INTER);
4399 %}
4400
4401 operand immILog()
4402 %{
4403 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4404 match(ConI);
4405
4406 op_cost(0);
4407 format %{ %}
4408 interface(CONST_INTER);
4409 %}
4410
4411 // Integer operands 64 bit
4412 // 64 bit immediate
4413 operand immL()
4414 %{
4415 match(ConL);
4416
4417 op_cost(0);
4418 format %{ %}
4419 interface(CONST_INTER);
4420 %}
4421
4422 // 64 bit zero
4423 operand immL0()
4424 %{
4425 predicate(n->get_long() == 0);
4426 match(ConL);
4427
4428 op_cost(0);
4429 format %{ %}
4430 interface(CONST_INTER);
4431 %}
4432
4433 // 64 bit unit decrement
4434 operand immL_M1()
4435 %{
4436 predicate(n->get_long() == -1);
4437 match(ConL);
4438
4439 op_cost(0);
4440 format %{ %}
4441 interface(CONST_INTER);
4442 %}
4443
4444 // 64 bit integer valid for add sub immediate
4445 operand immLAddSub()
4446 %{
4447 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4448 match(ConL);
4449 op_cost(0);
4450 format %{ %}
4451 interface(CONST_INTER);
4452 %}
4453
4454 // 64 bit integer valid for addv subv immediate
4455 operand immLAddSubV()
4456 %{
4457 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4458 match(ConL);
4459
4460 op_cost(0);
4461 format %{ %}
4462 interface(CONST_INTER);
4463 %}
4464
4465 // 64 bit integer valid for logical immediate
4466 operand immLLog()
4467 %{
4468 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4469 match(ConL);
4470 op_cost(0);
4471 format %{ %}
4472 interface(CONST_INTER);
4473 %}
4474
4475 // Long Immediate: low 32-bit mask
4476 operand immL_32bits()
4477 %{
4478 predicate(n->get_long() == 0xFFFFFFFFL);
4479 match(ConL);
4480 op_cost(0);
4481 format %{ %}
4482 interface(CONST_INTER);
4483 %}
4484
4485 // Pointer operands
4486 // Pointer Immediate
4487 operand immP()
4488 %{
4489 match(ConP);
4490
4491 op_cost(0);
4492 format %{ %}
4493 interface(CONST_INTER);
4494 %}
4495
4496 // nullptr Pointer Immediate
4497 operand immP0()
4498 %{
4499 predicate(n->get_ptr() == 0);
4500 match(ConP);
4501
4502 op_cost(0);
4503 format %{ %}
4504 interface(CONST_INTER);
4505 %}
4506
4507 // Pointer Immediate One
4508 // this is used in object initialization (initial object header)
4509 operand immP_1()
4510 %{
4511 predicate(n->get_ptr() == 1);
4512 match(ConP);
4513
4514 op_cost(0);
4515 format %{ %}
4516 interface(CONST_INTER);
4517 %}
4518
4519 // AOT Runtime Constants Address
4520 operand immAOTRuntimeConstantsAddress()
4521 %{
4522 // Check if the address is in the range of AOT Runtime Constants
4523 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4524 match(ConP);
4525
4526 op_cost(0);
4527 format %{ %}
4528 interface(CONST_INTER);
4529 %}
4530
4531 // Float and Double operands
4532 // Double Immediate
4533 operand immD()
4534 %{
4535 match(ConD);
4536 op_cost(0);
4537 format %{ %}
4538 interface(CONST_INTER);
4539 %}
4540
4541 // Double Immediate: +0.0d
4542 operand immD0()
4543 %{
4544 predicate(jlong_cast(n->getd()) == 0);
4545 match(ConD);
4546
4547 op_cost(0);
4548 format %{ %}
4549 interface(CONST_INTER);
4550 %}
4551
4552 // constant 'double +0.0'.
4553 operand immDPacked()
4554 %{
4555 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4556 match(ConD);
4557 op_cost(0);
4558 format %{ %}
4559 interface(CONST_INTER);
4560 %}
4561
4562 // Float Immediate
4563 operand immF()
4564 %{
4565 match(ConF);
4566 op_cost(0);
4567 format %{ %}
4568 interface(CONST_INTER);
4569 %}
4570
4571 // Float Immediate: +0.0f.
4572 operand immF0()
4573 %{
4574 predicate(jint_cast(n->getf()) == 0);
4575 match(ConF);
4576
4577 op_cost(0);
4578 format %{ %}
4579 interface(CONST_INTER);
4580 %}
4581
4582 // Half Float (FP16) Immediate
4583 operand immH()
4584 %{
4585 match(ConH);
4586 op_cost(0);
4587 format %{ %}
4588 interface(CONST_INTER);
4589 %}
4590
4591 //
4592 operand immFPacked()
4593 %{
4594 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4595 match(ConF);
4596 op_cost(0);
4597 format %{ %}
4598 interface(CONST_INTER);
4599 %}
4600
4601 // Narrow pointer operands
4602 // Narrow Pointer Immediate
4603 operand immN()
4604 %{
4605 match(ConN);
4606
4607 op_cost(0);
4608 format %{ %}
4609 interface(CONST_INTER);
4610 %}
4611
4612 // Narrow nullptr Pointer Immediate
4613 operand immN0()
4614 %{
4615 predicate(n->get_narrowcon() == 0);
4616 match(ConN);
4617
4618 op_cost(0);
4619 format %{ %}
4620 interface(CONST_INTER);
4621 %}
4622
4623 operand immNKlass()
4624 %{
4625 match(ConNKlass);
4626
4627 op_cost(0);
4628 format %{ %}
4629 interface(CONST_INTER);
4630 %}
4631
4632 // Integer 32 bit Register Operands
4633 // Integer 32 bitRegister (excludes SP)
4634 operand iRegI()
4635 %{
4636 constraint(ALLOC_IN_RC(any_reg32));
4637 match(RegI);
4638 match(iRegINoSp);
4639 op_cost(0);
4640 format %{ %}
4641 interface(REG_INTER);
4642 %}
4643
4644 // Integer 32 bit Register not Special
4645 operand iRegINoSp()
4646 %{
4647 constraint(ALLOC_IN_RC(no_special_reg32));
4648 match(RegI);
4649 op_cost(0);
4650 format %{ %}
4651 interface(REG_INTER);
4652 %}
4653
4654 // Integer 64 bit Register Operands
4655 // Integer 64 bit Register (includes SP)
4656 operand iRegL()
4657 %{
4658 constraint(ALLOC_IN_RC(any_reg));
4659 match(RegL);
4660 match(iRegLNoSp);
4661 op_cost(0);
4662 format %{ %}
4663 interface(REG_INTER);
4664 %}
4665
4666 // Integer 64 bit Register not Special
4667 operand iRegLNoSp()
4668 %{
4669 constraint(ALLOC_IN_RC(no_special_reg));
4670 match(RegL);
4671 match(iRegL_R0);
4672 format %{ %}
4673 interface(REG_INTER);
4674 %}
4675
4676 // Pointer Register Operands
4677 // Pointer Register
4678 operand iRegP()
4679 %{
4680 constraint(ALLOC_IN_RC(ptr_reg));
4681 match(RegP);
4682 match(iRegPNoSp);
4683 match(iRegP_R0);
4684 //match(iRegP_R2);
4685 //match(iRegP_R4);
4686 match(iRegP_R5);
4687 match(thread_RegP);
4688 op_cost(0);
4689 format %{ %}
4690 interface(REG_INTER);
4691 %}
4692
4693 // Pointer 64 bit Register not Special
4694 operand iRegPNoSp()
4695 %{
4696 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4697 match(RegP);
4698 // match(iRegP);
4699 // match(iRegP_R0);
4700 // match(iRegP_R2);
4701 // match(iRegP_R4);
4702 // match(iRegP_R5);
4703 // match(thread_RegP);
4704 op_cost(0);
4705 format %{ %}
4706 interface(REG_INTER);
4707 %}
4708
4709 // This operand is not allowed to use rfp even if
4710 // rfp is not used to hold the frame pointer.
4711 operand iRegPNoSpNoRfp()
4712 %{
4713 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4714 match(RegP);
4715 match(iRegPNoSp);
4716 op_cost(0);
4717 format %{ %}
4718 interface(REG_INTER);
4719 %}
4720
4721 // Pointer 64 bit Register R0 only
4722 operand iRegP_R0()
4723 %{
4724 constraint(ALLOC_IN_RC(r0_reg));
4725 match(RegP);
4726 // match(iRegP);
4727 match(iRegPNoSp);
4728 op_cost(0);
4729 format %{ %}
4730 interface(REG_INTER);
4731 %}
4732
4733 // Pointer 64 bit Register R1 only
4734 operand iRegP_R1()
4735 %{
4736 constraint(ALLOC_IN_RC(r1_reg));
4737 match(RegP);
4738 // match(iRegP);
4739 match(iRegPNoSp);
4740 op_cost(0);
4741 format %{ %}
4742 interface(REG_INTER);
4743 %}
4744
4745 // Pointer 64 bit Register R2 only
4746 operand iRegP_R2()
4747 %{
4748 constraint(ALLOC_IN_RC(r2_reg));
4749 match(RegP);
4750 // match(iRegP);
4751 match(iRegPNoSp);
4752 op_cost(0);
4753 format %{ %}
4754 interface(REG_INTER);
4755 %}
4756
4757 // Pointer 64 bit Register R3 only
4758 operand iRegP_R3()
4759 %{
4760 constraint(ALLOC_IN_RC(r3_reg));
4761 match(RegP);
4762 // match(iRegP);
4763 match(iRegPNoSp);
4764 op_cost(0);
4765 format %{ %}
4766 interface(REG_INTER);
4767 %}
4768
4769 // Pointer 64 bit Register R4 only
4770 operand iRegP_R4()
4771 %{
4772 constraint(ALLOC_IN_RC(r4_reg));
4773 match(RegP);
4774 // match(iRegP);
4775 match(iRegPNoSp);
4776 op_cost(0);
4777 format %{ %}
4778 interface(REG_INTER);
4779 %}
4780
4781 // Pointer 64 bit Register R5 only
4782 operand iRegP_R5()
4783 %{
4784 constraint(ALLOC_IN_RC(r5_reg));
4785 match(RegP);
4786 // match(iRegP);
4787 match(iRegPNoSp);
4788 op_cost(0);
4789 format %{ %}
4790 interface(REG_INTER);
4791 %}
4792
4793 // Pointer 64 bit Register R10 only
4794 operand iRegP_R10()
4795 %{
4796 constraint(ALLOC_IN_RC(r10_reg));
4797 match(RegP);
4798 // match(iRegP);
4799 match(iRegPNoSp);
4800 op_cost(0);
4801 format %{ %}
4802 interface(REG_INTER);
4803 %}
4804
4805 // Long 64 bit Register R0 only
4806 operand iRegL_R0()
4807 %{
4808 constraint(ALLOC_IN_RC(r0_reg));
4809 match(RegL);
4810 match(iRegLNoSp);
4811 op_cost(0);
4812 format %{ %}
4813 interface(REG_INTER);
4814 %}
4815
4816 // Long 64 bit Register R11 only
4817 operand iRegL_R11()
4818 %{
4819 constraint(ALLOC_IN_RC(r11_reg));
4820 match(RegL);
4821 match(iRegLNoSp);
4822 op_cost(0);
4823 format %{ %}
4824 interface(REG_INTER);
4825 %}
4826
4827 // Register R0 only
4828 operand iRegI_R0()
4829 %{
4830 constraint(ALLOC_IN_RC(int_r0_reg));
4831 match(RegI);
4832 match(iRegINoSp);
4833 op_cost(0);
4834 format %{ %}
4835 interface(REG_INTER);
4836 %}
4837
4838 // Register R2 only
4839 operand iRegI_R2()
4840 %{
4841 constraint(ALLOC_IN_RC(int_r2_reg));
4842 match(RegI);
4843 match(iRegINoSp);
4844 op_cost(0);
4845 format %{ %}
4846 interface(REG_INTER);
4847 %}
4848
4849 // Register R3 only
4850 operand iRegI_R3()
4851 %{
4852 constraint(ALLOC_IN_RC(int_r3_reg));
4853 match(RegI);
4854 match(iRegINoSp);
4855 op_cost(0);
4856 format %{ %}
4857 interface(REG_INTER);
4858 %}
4859
4860
4861 // Register R4 only
4862 operand iRegI_R4()
4863 %{
4864 constraint(ALLOC_IN_RC(int_r4_reg));
4865 match(RegI);
4866 match(iRegINoSp);
4867 op_cost(0);
4868 format %{ %}
4869 interface(REG_INTER);
4870 %}
4871
4872
4873 // Pointer Register Operands
4874 // Narrow Pointer Register
4875 operand iRegN()
4876 %{
4877 constraint(ALLOC_IN_RC(any_reg32));
4878 match(RegN);
4879 match(iRegNNoSp);
4880 op_cost(0);
4881 format %{ %}
4882 interface(REG_INTER);
4883 %}
4884
4885 // Integer 64 bit Register not Special
4886 operand iRegNNoSp()
4887 %{
4888 constraint(ALLOC_IN_RC(no_special_reg32));
4889 match(RegN);
4890 op_cost(0);
4891 format %{ %}
4892 interface(REG_INTER);
4893 %}
4894
4895 // Float Register
4896 // Float register operands
4897 operand vRegF()
4898 %{
4899 constraint(ALLOC_IN_RC(float_reg));
4900 match(RegF);
4901
4902 op_cost(0);
4903 format %{ %}
4904 interface(REG_INTER);
4905 %}
4906
4907 // Double Register
4908 // Double register operands
4909 operand vRegD()
4910 %{
4911 constraint(ALLOC_IN_RC(double_reg));
4912 match(RegD);
4913
4914 op_cost(0);
4915 format %{ %}
4916 interface(REG_INTER);
4917 %}
4918
4919 // Generic vector class. This will be used for
4920 // all vector operands, including NEON and SVE.
4921 operand vReg()
4922 %{
4923 constraint(ALLOC_IN_RC(dynamic));
4924 match(VecA);
4925 match(VecD);
4926 match(VecX);
4927
4928 op_cost(0);
4929 format %{ %}
4930 interface(REG_INTER);
4931 %}
4932
4933 operand vReg_V10()
4934 %{
4935 constraint(ALLOC_IN_RC(v10_veca_reg));
4936 match(vReg);
4937
4938 op_cost(0);
4939 format %{ %}
4940 interface(REG_INTER);
4941 %}
4942
4943 operand vReg_V11()
4944 %{
4945 constraint(ALLOC_IN_RC(v11_veca_reg));
4946 match(vReg);
4947
4948 op_cost(0);
4949 format %{ %}
4950 interface(REG_INTER);
4951 %}
4952
4953 operand vReg_V12()
4954 %{
4955 constraint(ALLOC_IN_RC(v12_veca_reg));
4956 match(vReg);
4957
4958 op_cost(0);
4959 format %{ %}
4960 interface(REG_INTER);
4961 %}
4962
4963 operand vReg_V13()
4964 %{
4965 constraint(ALLOC_IN_RC(v13_veca_reg));
4966 match(vReg);
4967
4968 op_cost(0);
4969 format %{ %}
4970 interface(REG_INTER);
4971 %}
4972
4973 operand vReg_V17()
4974 %{
4975 constraint(ALLOC_IN_RC(v17_veca_reg));
4976 match(vReg);
4977
4978 op_cost(0);
4979 format %{ %}
4980 interface(REG_INTER);
4981 %}
4982
4983 operand vReg_V18()
4984 %{
4985 constraint(ALLOC_IN_RC(v18_veca_reg));
4986 match(vReg);
4987
4988 op_cost(0);
4989 format %{ %}
4990 interface(REG_INTER);
4991 %}
4992
4993 operand vReg_V23()
4994 %{
4995 constraint(ALLOC_IN_RC(v23_veca_reg));
4996 match(vReg);
4997
4998 op_cost(0);
4999 format %{ %}
5000 interface(REG_INTER);
5001 %}
5002
5003 operand vReg_V24()
5004 %{
5005 constraint(ALLOC_IN_RC(v24_veca_reg));
5006 match(vReg);
5007
5008 op_cost(0);
5009 format %{ %}
5010 interface(REG_INTER);
5011 %}
5012
5013 operand vecA()
5014 %{
5015 constraint(ALLOC_IN_RC(vectora_reg));
5016 match(VecA);
5017
5018 op_cost(0);
5019 format %{ %}
5020 interface(REG_INTER);
5021 %}
5022
5023 operand vecD()
5024 %{
5025 constraint(ALLOC_IN_RC(vectord_reg));
5026 match(VecD);
5027
5028 op_cost(0);
5029 format %{ %}
5030 interface(REG_INTER);
5031 %}
5032
5033 operand vecX()
5034 %{
5035 constraint(ALLOC_IN_RC(vectorx_reg));
5036 match(VecX);
5037
5038 op_cost(0);
5039 format %{ %}
5040 interface(REG_INTER);
5041 %}
5042
5043 operand vRegD_V0()
5044 %{
5045 constraint(ALLOC_IN_RC(v0_reg));
5046 match(RegD);
5047 op_cost(0);
5048 format %{ %}
5049 interface(REG_INTER);
5050 %}
5051
5052 operand vRegD_V1()
5053 %{
5054 constraint(ALLOC_IN_RC(v1_reg));
5055 match(RegD);
5056 op_cost(0);
5057 format %{ %}
5058 interface(REG_INTER);
5059 %}
5060
5061 operand vRegD_V2()
5062 %{
5063 constraint(ALLOC_IN_RC(v2_reg));
5064 match(RegD);
5065 op_cost(0);
5066 format %{ %}
5067 interface(REG_INTER);
5068 %}
5069
5070 operand vRegD_V3()
5071 %{
5072 constraint(ALLOC_IN_RC(v3_reg));
5073 match(RegD);
5074 op_cost(0);
5075 format %{ %}
5076 interface(REG_INTER);
5077 %}
5078
5079 operand vRegD_V4()
5080 %{
5081 constraint(ALLOC_IN_RC(v4_reg));
5082 match(RegD);
5083 op_cost(0);
5084 format %{ %}
5085 interface(REG_INTER);
5086 %}
5087
5088 operand vRegD_V5()
5089 %{
5090 constraint(ALLOC_IN_RC(v5_reg));
5091 match(RegD);
5092 op_cost(0);
5093 format %{ %}
5094 interface(REG_INTER);
5095 %}
5096
5097 operand vRegD_V6()
5098 %{
5099 constraint(ALLOC_IN_RC(v6_reg));
5100 match(RegD);
5101 op_cost(0);
5102 format %{ %}
5103 interface(REG_INTER);
5104 %}
5105
5106 operand vRegD_V7()
5107 %{
5108 constraint(ALLOC_IN_RC(v7_reg));
5109 match(RegD);
5110 op_cost(0);
5111 format %{ %}
5112 interface(REG_INTER);
5113 %}
5114
5115 operand vRegD_V12()
5116 %{
5117 constraint(ALLOC_IN_RC(v12_reg));
5118 match(RegD);
5119 op_cost(0);
5120 format %{ %}
5121 interface(REG_INTER);
5122 %}
5123
5124 operand vRegD_V13()
5125 %{
5126 constraint(ALLOC_IN_RC(v13_reg));
5127 match(RegD);
5128 op_cost(0);
5129 format %{ %}
5130 interface(REG_INTER);
5131 %}
5132
5133 operand pReg()
5134 %{
5135 constraint(ALLOC_IN_RC(pr_reg));
5136 match(RegVectMask);
5137 match(pRegGov);
5138 op_cost(0);
5139 format %{ %}
5140 interface(REG_INTER);
5141 %}
5142
5143 operand pRegGov()
5144 %{
5145 constraint(ALLOC_IN_RC(gov_pr));
5146 match(RegVectMask);
5147 match(pReg);
5148 op_cost(0);
5149 format %{ %}
5150 interface(REG_INTER);
5151 %}
5152
5153 operand pRegGov_P0()
5154 %{
5155 constraint(ALLOC_IN_RC(p0_reg));
5156 match(RegVectMask);
5157 op_cost(0);
5158 format %{ %}
5159 interface(REG_INTER);
5160 %}
5161
5162 operand pRegGov_P1()
5163 %{
5164 constraint(ALLOC_IN_RC(p1_reg));
5165 match(RegVectMask);
5166 op_cost(0);
5167 format %{ %}
5168 interface(REG_INTER);
5169 %}
5170
5171 // Flags register, used as output of signed compare instructions
5172
5173 // note that on AArch64 we also use this register as the output for
5174 // for floating point compare instructions (CmpF CmpD). this ensures
5175 // that ordered inequality tests use GT, GE, LT or LE none of which
5176 // pass through cases where the result is unordered i.e. one or both
5177 // inputs to the compare is a NaN. this means that the ideal code can
5178 // replace e.g. a GT with an LE and not end up capturing the NaN case
5179 // (where the comparison should always fail). EQ and NE tests are
5180 // always generated in ideal code so that unordered folds into the NE
5181 // case, matching the behaviour of AArch64 NE.
5182 //
5183 // This differs from x86 where the outputs of FP compares use a
5184 // special FP flags registers and where compares based on this
5185 // register are distinguished into ordered inequalities (cmpOpUCF) and
5186 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5187 // to explicitly handle the unordered case in branches. x86 also has
5188 // to include extra CMoveX rules to accept a cmpOpUCF input.
5189
5190 operand rFlagsReg()
5191 %{
5192 constraint(ALLOC_IN_RC(int_flags));
5193 match(RegFlags);
5194
5195 op_cost(0);
5196 format %{ "RFLAGS" %}
5197 interface(REG_INTER);
5198 %}
5199
5200 // Flags register, used as output of unsigned compare instructions
5201 operand rFlagsRegU()
5202 %{
5203 constraint(ALLOC_IN_RC(int_flags));
5204 match(RegFlags);
5205
5206 op_cost(0);
5207 format %{ "RFLAGSU" %}
5208 interface(REG_INTER);
5209 %}
5210
5211 // Special Registers
5212
5213 // Method Register
5214 operand inline_cache_RegP(iRegP reg)
5215 %{
5216 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5217 match(reg);
5218 match(iRegPNoSp);
5219 op_cost(0);
5220 format %{ %}
5221 interface(REG_INTER);
5222 %}
5223
5224 // Thread Register
5225 operand thread_RegP(iRegP reg)
5226 %{
5227 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5228 match(reg);
5229 op_cost(0);
5230 format %{ %}
5231 interface(REG_INTER);
5232 %}
5233
5234 //----------Memory Operands----------------------------------------------------
5235
5236 operand indirect(iRegP reg)
5237 %{
5238 constraint(ALLOC_IN_RC(ptr_reg));
5239 match(reg);
5240 op_cost(0);
5241 format %{ "[$reg]" %}
5242 interface(MEMORY_INTER) %{
5243 base($reg);
5244 index(0xffffffff);
5245 scale(0x0);
5246 disp(0x0);
5247 %}
5248 %}
5249
5250 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5251 %{
5252 constraint(ALLOC_IN_RC(ptr_reg));
5253 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5254 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5255 op_cost(0);
5256 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5257 interface(MEMORY_INTER) %{
5258 base($reg);
5259 index($ireg);
5260 scale($scale);
5261 disp(0x0);
5262 %}
5263 %}
5264
5265 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5266 %{
5267 constraint(ALLOC_IN_RC(ptr_reg));
5268 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5269 match(AddP reg (LShiftL lreg scale));
5270 op_cost(0);
5271 format %{ "$reg, $lreg lsl($scale)" %}
5272 interface(MEMORY_INTER) %{
5273 base($reg);
5274 index($lreg);
5275 scale($scale);
5276 disp(0x0);
5277 %}
5278 %}
5279
5280 operand indIndexI2L(iRegP reg, iRegI ireg)
5281 %{
5282 constraint(ALLOC_IN_RC(ptr_reg));
5283 match(AddP reg (ConvI2L ireg));
5284 op_cost(0);
5285 format %{ "$reg, $ireg, 0, I2L" %}
5286 interface(MEMORY_INTER) %{
5287 base($reg);
5288 index($ireg);
5289 scale(0x0);
5290 disp(0x0);
5291 %}
5292 %}
5293
5294 operand indIndex(iRegP reg, iRegL lreg)
5295 %{
5296 constraint(ALLOC_IN_RC(ptr_reg));
5297 match(AddP reg lreg);
5298 op_cost(0);
5299 format %{ "$reg, $lreg" %}
5300 interface(MEMORY_INTER) %{
5301 base($reg);
5302 index($lreg);
5303 scale(0x0);
5304 disp(0x0);
5305 %}
5306 %}
5307
5308 operand indOffI1(iRegP reg, immIOffset1 off)
5309 %{
5310 constraint(ALLOC_IN_RC(ptr_reg));
5311 match(AddP reg off);
5312 op_cost(0);
5313 format %{ "[$reg, $off]" %}
5314 interface(MEMORY_INTER) %{
5315 base($reg);
5316 index(0xffffffff);
5317 scale(0x0);
5318 disp($off);
5319 %}
5320 %}
5321
5322 operand indOffI2(iRegP reg, immIOffset2 off)
5323 %{
5324 constraint(ALLOC_IN_RC(ptr_reg));
5325 match(AddP reg off);
5326 op_cost(0);
5327 format %{ "[$reg, $off]" %}
5328 interface(MEMORY_INTER) %{
5329 base($reg);
5330 index(0xffffffff);
5331 scale(0x0);
5332 disp($off);
5333 %}
5334 %}
5335
5336 operand indOffI4(iRegP reg, immIOffset4 off)
5337 %{
5338 constraint(ALLOC_IN_RC(ptr_reg));
5339 match(AddP reg off);
5340 op_cost(0);
5341 format %{ "[$reg, $off]" %}
5342 interface(MEMORY_INTER) %{
5343 base($reg);
5344 index(0xffffffff);
5345 scale(0x0);
5346 disp($off);
5347 %}
5348 %}
5349
5350 operand indOffI8(iRegP reg, immIOffset8 off)
5351 %{
5352 constraint(ALLOC_IN_RC(ptr_reg));
5353 match(AddP reg off);
5354 op_cost(0);
5355 format %{ "[$reg, $off]" %}
5356 interface(MEMORY_INTER) %{
5357 base($reg);
5358 index(0xffffffff);
5359 scale(0x0);
5360 disp($off);
5361 %}
5362 %}
5363
5364 operand indOffI16(iRegP reg, immIOffset16 off)
5365 %{
5366 constraint(ALLOC_IN_RC(ptr_reg));
5367 match(AddP reg off);
5368 op_cost(0);
5369 format %{ "[$reg, $off]" %}
5370 interface(MEMORY_INTER) %{
5371 base($reg);
5372 index(0xffffffff);
5373 scale(0x0);
5374 disp($off);
5375 %}
5376 %}
5377
5378 operand indOffL1(iRegP reg, immLoffset1 off)
5379 %{
5380 constraint(ALLOC_IN_RC(ptr_reg));
5381 match(AddP reg off);
5382 op_cost(0);
5383 format %{ "[$reg, $off]" %}
5384 interface(MEMORY_INTER) %{
5385 base($reg);
5386 index(0xffffffff);
5387 scale(0x0);
5388 disp($off);
5389 %}
5390 %}
5391
5392 operand indOffL2(iRegP reg, immLoffset2 off)
5393 %{
5394 constraint(ALLOC_IN_RC(ptr_reg));
5395 match(AddP reg off);
5396 op_cost(0);
5397 format %{ "[$reg, $off]" %}
5398 interface(MEMORY_INTER) %{
5399 base($reg);
5400 index(0xffffffff);
5401 scale(0x0);
5402 disp($off);
5403 %}
5404 %}
5405
5406 operand indOffL4(iRegP reg, immLoffset4 off)
5407 %{
5408 constraint(ALLOC_IN_RC(ptr_reg));
5409 match(AddP reg off);
5410 op_cost(0);
5411 format %{ "[$reg, $off]" %}
5412 interface(MEMORY_INTER) %{
5413 base($reg);
5414 index(0xffffffff);
5415 scale(0x0);
5416 disp($off);
5417 %}
5418 %}
5419
5420 operand indOffL8(iRegP reg, immLoffset8 off)
5421 %{
5422 constraint(ALLOC_IN_RC(ptr_reg));
5423 match(AddP reg off);
5424 op_cost(0);
5425 format %{ "[$reg, $off]" %}
5426 interface(MEMORY_INTER) %{
5427 base($reg);
5428 index(0xffffffff);
5429 scale(0x0);
5430 disp($off);
5431 %}
5432 %}
5433
5434 operand indOffL16(iRegP reg, immLoffset16 off)
5435 %{
5436 constraint(ALLOC_IN_RC(ptr_reg));
5437 match(AddP reg off);
5438 op_cost(0);
5439 format %{ "[$reg, $off]" %}
5440 interface(MEMORY_INTER) %{
5441 base($reg);
5442 index(0xffffffff);
5443 scale(0x0);
5444 disp($off);
5445 %}
5446 %}
5447
5448 operand indirectX2P(iRegL reg)
5449 %{
5450 constraint(ALLOC_IN_RC(ptr_reg));
5451 match(CastX2P reg);
5452 op_cost(0);
5453 format %{ "[$reg]\t# long -> ptr" %}
5454 interface(MEMORY_INTER) %{
5455 base($reg);
5456 index(0xffffffff);
5457 scale(0x0);
5458 disp(0x0);
5459 %}
5460 %}
5461
5462 operand indOffX2P(iRegL reg, immLOffset off)
5463 %{
5464 constraint(ALLOC_IN_RC(ptr_reg));
5465 match(AddP (CastX2P reg) off);
5466 op_cost(0);
5467 format %{ "[$reg, $off]\t# long -> ptr" %}
5468 interface(MEMORY_INTER) %{
5469 base($reg);
5470 index(0xffffffff);
5471 scale(0x0);
5472 disp($off);
5473 %}
5474 %}
5475
5476 operand indirectN(iRegN reg)
5477 %{
5478 predicate(CompressedOops::shift() == 0);
5479 constraint(ALLOC_IN_RC(ptr_reg));
5480 match(DecodeN reg);
5481 op_cost(0);
5482 format %{ "[$reg]\t# narrow" %}
5483 interface(MEMORY_INTER) %{
5484 base($reg);
5485 index(0xffffffff);
5486 scale(0x0);
5487 disp(0x0);
5488 %}
5489 %}
5490
5491 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5492 %{
5493 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5494 constraint(ALLOC_IN_RC(ptr_reg));
5495 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5496 op_cost(0);
5497 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5498 interface(MEMORY_INTER) %{
5499 base($reg);
5500 index($ireg);
5501 scale($scale);
5502 disp(0x0);
5503 %}
5504 %}
5505
5506 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5507 %{
5508 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5509 constraint(ALLOC_IN_RC(ptr_reg));
5510 match(AddP (DecodeN reg) (LShiftL lreg scale));
5511 op_cost(0);
5512 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5513 interface(MEMORY_INTER) %{
5514 base($reg);
5515 index($lreg);
5516 scale($scale);
5517 disp(0x0);
5518 %}
5519 %}
5520
5521 operand indIndexI2LN(iRegN reg, iRegI ireg)
5522 %{
5523 predicate(CompressedOops::shift() == 0);
5524 constraint(ALLOC_IN_RC(ptr_reg));
5525 match(AddP (DecodeN reg) (ConvI2L ireg));
5526 op_cost(0);
5527 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5528 interface(MEMORY_INTER) %{
5529 base($reg);
5530 index($ireg);
5531 scale(0x0);
5532 disp(0x0);
5533 %}
5534 %}
5535
5536 operand indIndexN(iRegN reg, iRegL lreg)
5537 %{
5538 predicate(CompressedOops::shift() == 0);
5539 constraint(ALLOC_IN_RC(ptr_reg));
5540 match(AddP (DecodeN reg) lreg);
5541 op_cost(0);
5542 format %{ "$reg, $lreg\t# narrow" %}
5543 interface(MEMORY_INTER) %{
5544 base($reg);
5545 index($lreg);
5546 scale(0x0);
5547 disp(0x0);
5548 %}
5549 %}
5550
5551 operand indOffIN(iRegN reg, immIOffset off)
5552 %{
5553 predicate(CompressedOops::shift() == 0);
5554 constraint(ALLOC_IN_RC(ptr_reg));
5555 match(AddP (DecodeN reg) off);
5556 op_cost(0);
5557 format %{ "[$reg, $off]\t# narrow" %}
5558 interface(MEMORY_INTER) %{
5559 base($reg);
5560 index(0xffffffff);
5561 scale(0x0);
5562 disp($off);
5563 %}
5564 %}
5565
5566 operand indOffLN(iRegN reg, immLOffset off)
5567 %{
5568 predicate(CompressedOops::shift() == 0);
5569 constraint(ALLOC_IN_RC(ptr_reg));
5570 match(AddP (DecodeN reg) off);
5571 op_cost(0);
5572 format %{ "[$reg, $off]\t# narrow" %}
5573 interface(MEMORY_INTER) %{
5574 base($reg);
5575 index(0xffffffff);
5576 scale(0x0);
5577 disp($off);
5578 %}
5579 %}
5580
5581
5582 //----------Special Memory Operands--------------------------------------------
5583 // Stack Slot Operand - This operand is used for loading and storing temporary
5584 // values on the stack where a match requires a value to
5585 // flow through memory.
5586 operand stackSlotP(sRegP reg)
5587 %{
5588 constraint(ALLOC_IN_RC(stack_slots));
5589 op_cost(100);
5590 // No match rule because this operand is only generated in matching
5591 // match(RegP);
5592 format %{ "[$reg]" %}
5593 interface(MEMORY_INTER) %{
5594 base(0x1e); // RSP
5595 index(0x0); // No Index
5596 scale(0x0); // No Scale
5597 disp($reg); // Stack Offset
5598 %}
5599 %}
5600
5601 operand stackSlotI(sRegI reg)
5602 %{
5603 constraint(ALLOC_IN_RC(stack_slots));
5604 // No match rule because this operand is only generated in matching
5605 // match(RegI);
5606 format %{ "[$reg]" %}
5607 interface(MEMORY_INTER) %{
5608 base(0x1e); // RSP
5609 index(0x0); // No Index
5610 scale(0x0); // No Scale
5611 disp($reg); // Stack Offset
5612 %}
5613 %}
5614
5615 operand stackSlotF(sRegF reg)
5616 %{
5617 constraint(ALLOC_IN_RC(stack_slots));
5618 // No match rule because this operand is only generated in matching
5619 // match(RegF);
5620 format %{ "[$reg]" %}
5621 interface(MEMORY_INTER) %{
5622 base(0x1e); // RSP
5623 index(0x0); // No Index
5624 scale(0x0); // No Scale
5625 disp($reg); // Stack Offset
5626 %}
5627 %}
5628
5629 operand stackSlotD(sRegD reg)
5630 %{
5631 constraint(ALLOC_IN_RC(stack_slots));
5632 // No match rule because this operand is only generated in matching
5633 // match(RegD);
5634 format %{ "[$reg]" %}
5635 interface(MEMORY_INTER) %{
5636 base(0x1e); // RSP
5637 index(0x0); // No Index
5638 scale(0x0); // No Scale
5639 disp($reg); // Stack Offset
5640 %}
5641 %}
5642
5643 operand stackSlotL(sRegL reg)
5644 %{
5645 constraint(ALLOC_IN_RC(stack_slots));
5646 // No match rule because this operand is only generated in matching
5647 // match(RegL);
5648 format %{ "[$reg]" %}
5649 interface(MEMORY_INTER) %{
5650 base(0x1e); // RSP
5651 index(0x0); // No Index
5652 scale(0x0); // No Scale
5653 disp($reg); // Stack Offset
5654 %}
5655 %}
5656
5657 // Operands for expressing Control Flow
5658 // NOTE: Label is a predefined operand which should not be redefined in
5659 // the AD file. It is generically handled within the ADLC.
5660
5661 //----------Conditional Branch Operands----------------------------------------
5662 // Comparison Op - This is the operation of the comparison, and is limited to
5663 // the following set of codes:
5664 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5665 //
5666 // Other attributes of the comparison, such as unsignedness, are specified
5667 // by the comparison instruction that sets a condition code flags register.
5668 // That result is represented by a flags operand whose subtype is appropriate
5669 // to the unsignedness (etc.) of the comparison.
5670 //
5671 // Later, the instruction which matches both the Comparison Op (a Bool) and
5672 // the flags (produced by the Cmp) specifies the coding of the comparison op
5673 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5674
5675 // used for signed integral comparisons and fp comparisons
5676
5677 operand cmpOp()
5678 %{
5679 match(Bool);
5680
5681 format %{ "" %}
5682 interface(COND_INTER) %{
5683 equal(0x0, "eq");
5684 not_equal(0x1, "ne");
5685 less(0xb, "lt");
5686 greater_equal(0xa, "ge");
5687 less_equal(0xd, "le");
5688 greater(0xc, "gt");
5689 overflow(0x6, "vs");
5690 no_overflow(0x7, "vc");
5691 %}
5692 %}
5693
5694 // used for unsigned integral comparisons
5695
5696 operand cmpOpU()
5697 %{
5698 match(Bool);
5699
5700 format %{ "" %}
5701 interface(COND_INTER) %{
5702 equal(0x0, "eq");
5703 not_equal(0x1, "ne");
5704 less(0x3, "lo");
5705 greater_equal(0x2, "hs");
5706 less_equal(0x9, "ls");
5707 greater(0x8, "hi");
5708 overflow(0x6, "vs");
5709 no_overflow(0x7, "vc");
5710 %}
5711 %}
5712
5713 // used for certain integral comparisons which can be
5714 // converted to cbxx or tbxx instructions
5715
5716 operand cmpOpEqNe()
5717 %{
5718 match(Bool);
5719 op_cost(0);
5720 predicate(n->as_Bool()->_test._test == BoolTest::ne
5721 || n->as_Bool()->_test._test == BoolTest::eq);
5722
5723 format %{ "" %}
5724 interface(COND_INTER) %{
5725 equal(0x0, "eq");
5726 not_equal(0x1, "ne");
5727 less(0xb, "lt");
5728 greater_equal(0xa, "ge");
5729 less_equal(0xd, "le");
5730 greater(0xc, "gt");
5731 overflow(0x6, "vs");
5732 no_overflow(0x7, "vc");
5733 %}
5734 %}
5735
5736 // used for certain integral comparisons which can be
5737 // converted to cbxx or tbxx instructions
5738
5739 operand cmpOpLtGe()
5740 %{
5741 match(Bool);
5742 op_cost(0);
5743
5744 predicate(n->as_Bool()->_test._test == BoolTest::lt
5745 || n->as_Bool()->_test._test == BoolTest::ge);
5746
5747 format %{ "" %}
5748 interface(COND_INTER) %{
5749 equal(0x0, "eq");
5750 not_equal(0x1, "ne");
5751 less(0xb, "lt");
5752 greater_equal(0xa, "ge");
5753 less_equal(0xd, "le");
5754 greater(0xc, "gt");
5755 overflow(0x6, "vs");
5756 no_overflow(0x7, "vc");
5757 %}
5758 %}
5759
5760 // used for certain unsigned integral comparisons which can be
5761 // converted to cbxx or tbxx instructions
5762
5763 operand cmpOpUEqNeLeGt()
5764 %{
5765 match(Bool);
5766 op_cost(0);
5767
5768 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5769 n->as_Bool()->_test._test == BoolTest::ne ||
5770 n->as_Bool()->_test._test == BoolTest::le ||
5771 n->as_Bool()->_test._test == BoolTest::gt);
5772
5773 format %{ "" %}
5774 interface(COND_INTER) %{
5775 equal(0x0, "eq");
5776 not_equal(0x1, "ne");
5777 less(0x3, "lo");
5778 greater_equal(0x2, "hs");
5779 less_equal(0x9, "ls");
5780 greater(0x8, "hi");
5781 overflow(0x6, "vs");
5782 no_overflow(0x7, "vc");
5783 %}
5784 %}
5785
5786 // Special operand allowing long args to int ops to be truncated for free
5787
5788 operand iRegL2I(iRegL reg) %{
5789
5790 op_cost(0);
5791
5792 match(ConvL2I reg);
5793
5794 format %{ "l2i($reg)" %}
5795
5796 interface(REG_INTER)
5797 %}
5798
5799 operand iRegL2P(iRegL reg) %{
5800
5801 op_cost(0);
5802
5803 match(CastX2P reg);
5804
5805 format %{ "l2p($reg)" %}
5806
5807 interface(REG_INTER)
5808 %}
5809
5810 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5811 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5812 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5813 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5814
5815 //----------OPERAND CLASSES----------------------------------------------------
5816 // Operand Classes are groups of operands that are used as to simplify
5817 // instruction definitions by not requiring the AD writer to specify
5818 // separate instructions for every form of operand when the
5819 // instruction accepts multiple operand types with the same basic
5820 // encoding and format. The classic case of this is memory operands.
5821
5822 // memory is used to define read/write location for load/store
5823 // instruction defs. we can turn a memory op into an Address
5824
5825 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5826 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5827
5828 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5829 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5830
5831 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5832 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5833
5834 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5835 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5836
5837 // All of the memory operands. For the pipeline description.
5838 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5839 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5840 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5841
5842
5843 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5844 // operations. it allows the src to be either an iRegI or a (ConvL2I
5845 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5846 // can be elided because the 32-bit instruction will just employ the
5847 // lower 32 bits anyway.
5848 //
5849 // n.b. this does not elide all L2I conversions. if the truncated
5850 // value is consumed by more than one operation then the ConvL2I
5851 // cannot be bundled into the consuming nodes so an l2i gets planted
5852 // (actually a movw $dst $src) and the downstream instructions consume
5853 // the result of the l2i as an iRegI input. That's a shame since the
5854 // movw is actually redundant but its not too costly.
5855
5856 opclass iRegIorL2I(iRegI, iRegL2I);
5857 opclass iRegPorL2P(iRegP, iRegL2P);
5858
5859 //----------PIPELINE-----------------------------------------------------------
5860 // Rules which define the behavior of the target architectures pipeline.
5861
5862 // For specific pipelines, eg A53, define the stages of that pipeline
5863 //pipe_desc(ISS, EX1, EX2, WR);
5864 #define ISS S0
5865 #define EX1 S1
5866 #define EX2 S2
5867 #define WR S3
5868
5869 // Integer ALU reg operation
5870 pipeline %{
5871
5872 attributes %{
5873 // ARM instructions are of fixed length
5874 fixed_size_instructions; // Fixed size instructions TODO does
5875 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5876 // ARM instructions come in 32-bit word units
5877 instruction_unit_size = 4; // An instruction is 4 bytes long
5878 instruction_fetch_unit_size = 64; // The processor fetches one line
5879 instruction_fetch_units = 1; // of 64 bytes
5880 %}
5881
5882 // We don't use an actual pipeline model so don't care about resources
5883 // or description. we do use pipeline classes to introduce fixed
5884 // latencies
5885
5886 //----------RESOURCES----------------------------------------------------------
5887 // Resources are the functional units available to the machine
5888
5889 resources( INS0, INS1, INS01 = INS0 | INS1,
5890 ALU0, ALU1, ALU = ALU0 | ALU1,
5891 MAC,
5892 DIV,
5893 BRANCH,
5894 LDST,
5895 NEON_FP);
5896
5897 //----------PIPELINE DESCRIPTION-----------------------------------------------
5898 // Pipeline Description specifies the stages in the machine's pipeline
5899
5900 // Define the pipeline as a generic 6 stage pipeline
5901 pipe_desc(S0, S1, S2, S3, S4, S5);
5902
5903 //----------PIPELINE CLASSES---------------------------------------------------
5904 // Pipeline Classes describe the stages in which input and output are
5905 // referenced by the hardware pipeline.
5906
5907 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5908 %{
5909 single_instruction;
5910 src1 : S1(read);
5911 src2 : S2(read);
5912 dst : S5(write);
5913 INS01 : ISS;
5914 NEON_FP : S5;
5915 %}
5916
5917 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5918 %{
5919 single_instruction;
5920 src1 : S1(read);
5921 src2 : S2(read);
5922 dst : S5(write);
5923 INS01 : ISS;
5924 NEON_FP : S5;
5925 %}
5926
5927 pipe_class fp_uop_s(vRegF dst, vRegF src)
5928 %{
5929 single_instruction;
5930 src : S1(read);
5931 dst : S5(write);
5932 INS01 : ISS;
5933 NEON_FP : S5;
5934 %}
5935
5936 pipe_class fp_uop_d(vRegD dst, vRegD src)
5937 %{
5938 single_instruction;
5939 src : S1(read);
5940 dst : S5(write);
5941 INS01 : ISS;
5942 NEON_FP : S5;
5943 %}
5944
5945 pipe_class fp_d2f(vRegF dst, vRegD src)
5946 %{
5947 single_instruction;
5948 src : S1(read);
5949 dst : S5(write);
5950 INS01 : ISS;
5951 NEON_FP : S5;
5952 %}
5953
5954 pipe_class fp_f2d(vRegD dst, vRegF src)
5955 %{
5956 single_instruction;
5957 src : S1(read);
5958 dst : S5(write);
5959 INS01 : ISS;
5960 NEON_FP : S5;
5961 %}
5962
5963 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5964 %{
5965 single_instruction;
5966 src : S1(read);
5967 dst : S5(write);
5968 INS01 : ISS;
5969 NEON_FP : S5;
5970 %}
5971
5972 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5973 %{
5974 single_instruction;
5975 src : S1(read);
5976 dst : S5(write);
5977 INS01 : ISS;
5978 NEON_FP : S5;
5979 %}
5980
5981 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5982 %{
5983 single_instruction;
5984 src : S1(read);
5985 dst : S5(write);
5986 INS01 : ISS;
5987 NEON_FP : S5;
5988 %}
5989
5990 pipe_class fp_l2f(vRegF dst, iRegL src)
5991 %{
5992 single_instruction;
5993 src : S1(read);
5994 dst : S5(write);
5995 INS01 : ISS;
5996 NEON_FP : S5;
5997 %}
5998
5999 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6000 %{
6001 single_instruction;
6002 src : S1(read);
6003 dst : S5(write);
6004 INS01 : ISS;
6005 NEON_FP : S5;
6006 %}
6007
6008 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6009 %{
6010 single_instruction;
6011 src : S1(read);
6012 dst : S5(write);
6013 INS01 : ISS;
6014 NEON_FP : S5;
6015 %}
6016
6017 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6018 %{
6019 single_instruction;
6020 src : S1(read);
6021 dst : S5(write);
6022 INS01 : ISS;
6023 NEON_FP : S5;
6024 %}
6025
6026 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6027 %{
6028 single_instruction;
6029 src : S1(read);
6030 dst : S5(write);
6031 INS01 : ISS;
6032 NEON_FP : S5;
6033 %}
6034
6035 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6036 %{
6037 single_instruction;
6038 src1 : S1(read);
6039 src2 : S2(read);
6040 dst : S5(write);
6041 INS0 : ISS;
6042 NEON_FP : S5;
6043 %}
6044
6045 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6046 %{
6047 single_instruction;
6048 src1 : S1(read);
6049 src2 : S2(read);
6050 dst : S5(write);
6051 INS0 : ISS;
6052 NEON_FP : S5;
6053 %}
6054
6055 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6056 %{
6057 single_instruction;
6058 cr : S1(read);
6059 src1 : S1(read);
6060 src2 : S1(read);
6061 dst : S3(write);
6062 INS01 : ISS;
6063 NEON_FP : S3;
6064 %}
6065
6066 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6067 %{
6068 single_instruction;
6069 cr : S1(read);
6070 src1 : S1(read);
6071 src2 : S1(read);
6072 dst : S3(write);
6073 INS01 : ISS;
6074 NEON_FP : S3;
6075 %}
6076
6077 pipe_class fp_imm_s(vRegF dst)
6078 %{
6079 single_instruction;
6080 dst : S3(write);
6081 INS01 : ISS;
6082 NEON_FP : S3;
6083 %}
6084
6085 pipe_class fp_imm_d(vRegD dst)
6086 %{
6087 single_instruction;
6088 dst : S3(write);
6089 INS01 : ISS;
6090 NEON_FP : S3;
6091 %}
6092
6093 pipe_class fp_load_constant_s(vRegF dst)
6094 %{
6095 single_instruction;
6096 dst : S4(write);
6097 INS01 : ISS;
6098 NEON_FP : S4;
6099 %}
6100
6101 pipe_class fp_load_constant_d(vRegD dst)
6102 %{
6103 single_instruction;
6104 dst : S4(write);
6105 INS01 : ISS;
6106 NEON_FP : S4;
6107 %}
6108
6109 //------- Integer ALU operations --------------------------
6110
6111 // Integer ALU reg-reg operation
6112 // Operands needed in EX1, result generated in EX2
6113 // Eg. ADD x0, x1, x2
6114 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6115 %{
6116 single_instruction;
6117 dst : EX2(write);
6118 src1 : EX1(read);
6119 src2 : EX1(read);
6120 INS01 : ISS; // Dual issue as instruction 0 or 1
6121 ALU : EX2;
6122 %}
6123
6124 // Integer ALU reg-reg operation with constant shift
6125 // Shifted register must be available in LATE_ISS instead of EX1
6126 // Eg. ADD x0, x1, x2, LSL #2
6127 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6128 %{
6129 single_instruction;
6130 dst : EX2(write);
6131 src1 : EX1(read);
6132 src2 : ISS(read);
6133 INS01 : ISS;
6134 ALU : EX2;
6135 %}
6136
6137 // Integer ALU reg operation with constant shift
6138 // Eg. LSL x0, x1, #shift
6139 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6140 %{
6141 single_instruction;
6142 dst : EX2(write);
6143 src1 : ISS(read);
6144 INS01 : ISS;
6145 ALU : EX2;
6146 %}
6147
6148 // Integer ALU reg-reg operation with variable shift
6149 // Both operands must be available in LATE_ISS instead of EX1
6150 // Result is available in EX1 instead of EX2
6151 // Eg. LSLV x0, x1, x2
6152 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6153 %{
6154 single_instruction;
6155 dst : EX1(write);
6156 src1 : ISS(read);
6157 src2 : ISS(read);
6158 INS01 : ISS;
6159 ALU : EX1;
6160 %}
6161
6162 // Integer ALU reg-reg operation with extract
6163 // As for _vshift above, but result generated in EX2
6164 // Eg. EXTR x0, x1, x2, #N
6165 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6166 %{
6167 single_instruction;
6168 dst : EX2(write);
6169 src1 : ISS(read);
6170 src2 : ISS(read);
6171 INS1 : ISS; // Can only dual issue as Instruction 1
6172 ALU : EX1;
6173 %}
6174
6175 // Integer ALU reg operation
6176 // Eg. NEG x0, x1
6177 pipe_class ialu_reg(iRegI dst, iRegI src)
6178 %{
6179 single_instruction;
6180 dst : EX2(write);
6181 src : EX1(read);
6182 INS01 : ISS;
6183 ALU : EX2;
6184 %}
6185
6186 // Integer ALU reg mmediate operation
6187 // Eg. ADD x0, x1, #N
6188 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6189 %{
6190 single_instruction;
6191 dst : EX2(write);
6192 src1 : EX1(read);
6193 INS01 : ISS;
6194 ALU : EX2;
6195 %}
6196
6197 // Integer ALU immediate operation (no source operands)
6198 // Eg. MOV x0, #N
6199 pipe_class ialu_imm(iRegI dst)
6200 %{
6201 single_instruction;
6202 dst : EX1(write);
6203 INS01 : ISS;
6204 ALU : EX1;
6205 %}
6206
6207 //------- Compare operation -------------------------------
6208
6209 // Compare reg-reg
6210 // Eg. CMP x0, x1
6211 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6212 %{
6213 single_instruction;
6214 // fixed_latency(16);
6215 cr : EX2(write);
6216 op1 : EX1(read);
6217 op2 : EX1(read);
6218 INS01 : ISS;
6219 ALU : EX2;
6220 %}
6221
6222 // Compare reg-reg
6223 // Eg. CMP x0, #N
6224 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6225 %{
6226 single_instruction;
6227 // fixed_latency(16);
6228 cr : EX2(write);
6229 op1 : EX1(read);
6230 INS01 : ISS;
6231 ALU : EX2;
6232 %}
6233
6234 //------- Conditional instructions ------------------------
6235
6236 // Conditional no operands
6237 // Eg. CSINC x0, zr, zr, <cond>
6238 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6239 %{
6240 single_instruction;
6241 cr : EX1(read);
6242 dst : EX2(write);
6243 INS01 : ISS;
6244 ALU : EX2;
6245 %}
6246
6247 // Conditional 2 operand
6248 // EG. CSEL X0, X1, X2, <cond>
6249 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6250 %{
6251 single_instruction;
6252 cr : EX1(read);
6253 src1 : EX1(read);
6254 src2 : EX1(read);
6255 dst : EX2(write);
6256 INS01 : ISS;
6257 ALU : EX2;
6258 %}
6259
6260 // Conditional 2 operand
6261 // EG. CSEL X0, X1, X2, <cond>
6262 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6263 %{
6264 single_instruction;
6265 cr : EX1(read);
6266 src : EX1(read);
6267 dst : EX2(write);
6268 INS01 : ISS;
6269 ALU : EX2;
6270 %}
6271
6272 //------- Multiply pipeline operations --------------------
6273
6274 // Multiply reg-reg
6275 // Eg. MUL w0, w1, w2
6276 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6277 %{
6278 single_instruction;
6279 dst : WR(write);
6280 src1 : ISS(read);
6281 src2 : ISS(read);
6282 INS01 : ISS;
6283 MAC : WR;
6284 %}
6285
6286 // Multiply accumulate
6287 // Eg. MADD w0, w1, w2, w3
6288 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6289 %{
6290 single_instruction;
6291 dst : WR(write);
6292 src1 : ISS(read);
6293 src2 : ISS(read);
6294 src3 : ISS(read);
6295 INS01 : ISS;
6296 MAC : WR;
6297 %}
6298
6299 // Eg. MUL w0, w1, w2
6300 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6301 %{
6302 single_instruction;
6303 fixed_latency(3); // Maximum latency for 64 bit mul
6304 dst : WR(write);
6305 src1 : ISS(read);
6306 src2 : ISS(read);
6307 INS01 : ISS;
6308 MAC : WR;
6309 %}
6310
6311 // Multiply accumulate
6312 // Eg. MADD w0, w1, w2, w3
6313 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6314 %{
6315 single_instruction;
6316 fixed_latency(3); // Maximum latency for 64 bit mul
6317 dst : WR(write);
6318 src1 : ISS(read);
6319 src2 : ISS(read);
6320 src3 : ISS(read);
6321 INS01 : ISS;
6322 MAC : WR;
6323 %}
6324
6325 //------- Divide pipeline operations --------------------
6326
6327 // Eg. SDIV w0, w1, w2
6328 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6329 %{
6330 single_instruction;
6331 fixed_latency(8); // Maximum latency for 32 bit divide
6332 dst : WR(write);
6333 src1 : ISS(read);
6334 src2 : ISS(read);
6335 INS0 : ISS; // Can only dual issue as instruction 0
6336 DIV : WR;
6337 %}
6338
6339 // Eg. SDIV x0, x1, x2
6340 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6341 %{
6342 single_instruction;
6343 fixed_latency(16); // Maximum latency for 64 bit divide
6344 dst : WR(write);
6345 src1 : ISS(read);
6346 src2 : ISS(read);
6347 INS0 : ISS; // Can only dual issue as instruction 0
6348 DIV : WR;
6349 %}
6350
6351 //------- Load pipeline operations ------------------------
6352
6353 // Load - prefetch
6354 // Eg. PFRM <mem>
6355 pipe_class iload_prefetch(memory mem)
6356 %{
6357 single_instruction;
6358 mem : ISS(read);
6359 INS01 : ISS;
6360 LDST : WR;
6361 %}
6362
6363 // Load - reg, mem
6364 // Eg. LDR x0, <mem>
6365 pipe_class iload_reg_mem(iRegI dst, memory mem)
6366 %{
6367 single_instruction;
6368 dst : WR(write);
6369 mem : ISS(read);
6370 INS01 : ISS;
6371 LDST : WR;
6372 %}
6373
6374 // Load - reg, reg
6375 // Eg. LDR x0, [sp, x1]
6376 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6377 %{
6378 single_instruction;
6379 dst : WR(write);
6380 src : ISS(read);
6381 INS01 : ISS;
6382 LDST : WR;
6383 %}
6384
6385 //------- Store pipeline operations -----------------------
6386
6387 // Store - zr, mem
6388 // Eg. STR zr, <mem>
6389 pipe_class istore_mem(memory mem)
6390 %{
6391 single_instruction;
6392 mem : ISS(read);
6393 INS01 : ISS;
6394 LDST : WR;
6395 %}
6396
6397 // Store - reg, mem
6398 // Eg. STR x0, <mem>
6399 pipe_class istore_reg_mem(iRegI src, memory mem)
6400 %{
6401 single_instruction;
6402 mem : ISS(read);
6403 src : EX2(read);
6404 INS01 : ISS;
6405 LDST : WR;
6406 %}
6407
6408 // Store - reg, reg
6409 // Eg. STR x0, [sp, x1]
6410 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6411 %{
6412 single_instruction;
6413 dst : ISS(read);
6414 src : EX2(read);
6415 INS01 : ISS;
6416 LDST : WR;
6417 %}
6418
6419 //------- Store pipeline operations -----------------------
6420
6421 // Branch
6422 pipe_class pipe_branch()
6423 %{
6424 single_instruction;
6425 INS01 : ISS;
6426 BRANCH : EX1;
6427 %}
6428
6429 // Conditional branch
6430 pipe_class pipe_branch_cond(rFlagsReg cr)
6431 %{
6432 single_instruction;
6433 cr : EX1(read);
6434 INS01 : ISS;
6435 BRANCH : EX1;
6436 %}
6437
6438 // Compare & Branch
6439 // EG. CBZ/CBNZ
6440 pipe_class pipe_cmp_branch(iRegI op1)
6441 %{
6442 single_instruction;
6443 op1 : EX1(read);
6444 INS01 : ISS;
6445 BRANCH : EX1;
6446 %}
6447
6448 //------- Synchronisation operations ----------------------
6449
6450 // Any operation requiring serialization.
6451 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6452 pipe_class pipe_serial()
6453 %{
6454 single_instruction;
6455 force_serialization;
6456 fixed_latency(16);
6457 INS01 : ISS(2); // Cannot dual issue with any other instruction
6458 LDST : WR;
6459 %}
6460
6461 // Generic big/slow expanded idiom - also serialized
6462 pipe_class pipe_slow()
6463 %{
6464 instruction_count(10);
6465 multiple_bundles;
6466 force_serialization;
6467 fixed_latency(16);
6468 INS01 : ISS(2); // Cannot dual issue with any other instruction
6469 LDST : WR;
6470 %}
6471
6472 // Empty pipeline class
6473 pipe_class pipe_class_empty()
6474 %{
6475 single_instruction;
6476 fixed_latency(0);
6477 %}
6478
6479 // Default pipeline class.
6480 pipe_class pipe_class_default()
6481 %{
6482 single_instruction;
6483 fixed_latency(2);
6484 %}
6485
6486 // Pipeline class for compares.
6487 pipe_class pipe_class_compare()
6488 %{
6489 single_instruction;
6490 fixed_latency(16);
6491 %}
6492
6493 // Pipeline class for memory operations.
6494 pipe_class pipe_class_memory()
6495 %{
6496 single_instruction;
6497 fixed_latency(16);
6498 %}
6499
6500 // Pipeline class for call.
6501 pipe_class pipe_class_call()
6502 %{
6503 single_instruction;
6504 fixed_latency(100);
6505 %}
6506
6507 // Define the class for the Nop node.
6508 define %{
6509 MachNop = pipe_class_empty;
6510 %}
6511
6512 %}
6513 //----------INSTRUCTIONS-------------------------------------------------------
6514 //
6515 // match -- States which machine-independent subtree may be replaced
6516 // by this instruction.
6517 // ins_cost -- The estimated cost of this instruction is used by instruction
6518 // selection to identify a minimum cost tree of machine
6519 // instructions that matches a tree of machine-independent
6520 // instructions.
6521 // format -- A string providing the disassembly for this instruction.
6522 // The value of an instruction's operand may be inserted
6523 // by referring to it with a '$' prefix.
6524 // opcode -- Three instruction opcodes may be provided. These are referred
6525 // to within an encode class as $primary, $secondary, and $tertiary
6526 // rrspectively. The primary opcode is commonly used to
6527 // indicate the type of machine instruction, while secondary
6528 // and tertiary are often used for prefix options or addressing
6529 // modes.
6530 // ins_encode -- A list of encode classes with parameters. The encode class
6531 // name must have been defined in an 'enc_class' specification
6532 // in the encode section of the architecture description.
6533
6534 // ============================================================================
6535 // Memory (Load/Store) Instructions
6536
6537 // Load Instructions
6538
6539 // Load Byte (8 bit signed)
6540 instruct loadB(iRegINoSp dst, memory1 mem)
6541 %{
6542 match(Set dst (LoadB mem));
6543 predicate(!needs_acquiring_load(n));
6544
6545 ins_cost(4 * INSN_COST);
6546 format %{ "ldrsbw $dst, $mem\t# byte" %}
6547
6548 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6549
6550 ins_pipe(iload_reg_mem);
6551 %}
6552
6553 // Load Byte (8 bit signed) into long
6554 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6555 %{
6556 match(Set dst (ConvI2L (LoadB mem)));
6557 predicate(!needs_acquiring_load(n->in(1)));
6558
6559 ins_cost(4 * INSN_COST);
6560 format %{ "ldrsb $dst, $mem\t# byte" %}
6561
6562 ins_encode(aarch64_enc_ldrsb(dst, mem));
6563
6564 ins_pipe(iload_reg_mem);
6565 %}
6566
6567 // Load Byte (8 bit unsigned)
6568 instruct loadUB(iRegINoSp dst, memory1 mem)
6569 %{
6570 match(Set dst (LoadUB mem));
6571 predicate(!needs_acquiring_load(n));
6572
6573 ins_cost(4 * INSN_COST);
6574 format %{ "ldrbw $dst, $mem\t# byte" %}
6575
6576 ins_encode(aarch64_enc_ldrb(dst, mem));
6577
6578 ins_pipe(iload_reg_mem);
6579 %}
6580
6581 // Load Byte (8 bit unsigned) into long
6582 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6583 %{
6584 match(Set dst (ConvI2L (LoadUB mem)));
6585 predicate(!needs_acquiring_load(n->in(1)));
6586
6587 ins_cost(4 * INSN_COST);
6588 format %{ "ldrb $dst, $mem\t# byte" %}
6589
6590 ins_encode(aarch64_enc_ldrb(dst, mem));
6591
6592 ins_pipe(iload_reg_mem);
6593 %}
6594
6595 // Load Short (16 bit signed)
6596 instruct loadS(iRegINoSp dst, memory2 mem)
6597 %{
6598 match(Set dst (LoadS mem));
6599 predicate(!needs_acquiring_load(n));
6600
6601 ins_cost(4 * INSN_COST);
6602 format %{ "ldrshw $dst, $mem\t# short" %}
6603
6604 ins_encode(aarch64_enc_ldrshw(dst, mem));
6605
6606 ins_pipe(iload_reg_mem);
6607 %}
6608
6609 // Load Short (16 bit signed) into long
6610 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6611 %{
6612 match(Set dst (ConvI2L (LoadS mem)));
6613 predicate(!needs_acquiring_load(n->in(1)));
6614
6615 ins_cost(4 * INSN_COST);
6616 format %{ "ldrsh $dst, $mem\t# short" %}
6617
6618 ins_encode(aarch64_enc_ldrsh(dst, mem));
6619
6620 ins_pipe(iload_reg_mem);
6621 %}
6622
6623 // Load Char (16 bit unsigned)
6624 instruct loadUS(iRegINoSp dst, memory2 mem)
6625 %{
6626 match(Set dst (LoadUS mem));
6627 predicate(!needs_acquiring_load(n));
6628
6629 ins_cost(4 * INSN_COST);
6630 format %{ "ldrh $dst, $mem\t# short" %}
6631
6632 ins_encode(aarch64_enc_ldrh(dst, mem));
6633
6634 ins_pipe(iload_reg_mem);
6635 %}
6636
6637 // Load Short/Char (16 bit unsigned) into long
6638 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6639 %{
6640 match(Set dst (ConvI2L (LoadUS mem)));
6641 predicate(!needs_acquiring_load(n->in(1)));
6642
6643 ins_cost(4 * INSN_COST);
6644 format %{ "ldrh $dst, $mem\t# short" %}
6645
6646 ins_encode(aarch64_enc_ldrh(dst, mem));
6647
6648 ins_pipe(iload_reg_mem);
6649 %}
6650
6651 // Load Integer (32 bit signed)
6652 instruct loadI(iRegINoSp dst, memory4 mem)
6653 %{
6654 match(Set dst (LoadI mem));
6655 predicate(!needs_acquiring_load(n));
6656
6657 ins_cost(4 * INSN_COST);
6658 format %{ "ldrw $dst, $mem\t# int" %}
6659
6660 ins_encode(aarch64_enc_ldrw(dst, mem));
6661
6662 ins_pipe(iload_reg_mem);
6663 %}
6664
6665 // Load Integer (32 bit signed) into long
6666 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6667 %{
6668 match(Set dst (ConvI2L (LoadI mem)));
6669 predicate(!needs_acquiring_load(n->in(1)));
6670
6671 ins_cost(4 * INSN_COST);
6672 format %{ "ldrsw $dst, $mem\t# int" %}
6673
6674 ins_encode(aarch64_enc_ldrsw(dst, mem));
6675
6676 ins_pipe(iload_reg_mem);
6677 %}
6678
6679 // Load Integer (32 bit unsigned) into long
6680 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6681 %{
6682 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6683 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6684
6685 ins_cost(4 * INSN_COST);
6686 format %{ "ldrw $dst, $mem\t# int" %}
6687
6688 ins_encode(aarch64_enc_ldrw(dst, mem));
6689
6690 ins_pipe(iload_reg_mem);
6691 %}
6692
6693 // Load Long (64 bit signed)
6694 instruct loadL(iRegLNoSp dst, memory8 mem)
6695 %{
6696 match(Set dst (LoadL mem));
6697 predicate(!needs_acquiring_load(n));
6698
6699 ins_cost(4 * INSN_COST);
6700 format %{ "ldr $dst, $mem\t# int" %}
6701
6702 ins_encode(aarch64_enc_ldr(dst, mem));
6703
6704 ins_pipe(iload_reg_mem);
6705 %}
6706
6707 // Load Range
6708 instruct loadRange(iRegINoSp dst, memory4 mem)
6709 %{
6710 match(Set dst (LoadRange mem));
6711
6712 ins_cost(4 * INSN_COST);
6713 format %{ "ldrw $dst, $mem\t# range" %}
6714
6715 ins_encode(aarch64_enc_ldrw(dst, mem));
6716
6717 ins_pipe(iload_reg_mem);
6718 %}
6719
6720 // Load Pointer
6721 instruct loadP(iRegPNoSp dst, memory8 mem)
6722 %{
6723 match(Set dst (LoadP mem));
6724 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6725
6726 ins_cost(4 * INSN_COST);
6727 format %{ "ldr $dst, $mem\t# ptr" %}
6728
6729 ins_encode(aarch64_enc_ldr(dst, mem));
6730
6731 ins_pipe(iload_reg_mem);
6732 %}
6733
6734 // Load Compressed Pointer
6735 instruct loadN(iRegNNoSp dst, memory4 mem)
6736 %{
6737 match(Set dst (LoadN mem));
6738 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6739
6740 ins_cost(4 * INSN_COST);
6741 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6742
6743 ins_encode(aarch64_enc_ldrw(dst, mem));
6744
6745 ins_pipe(iload_reg_mem);
6746 %}
6747
6748 // Load Klass Pointer
6749 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6750 %{
6751 match(Set dst (LoadKlass mem));
6752 predicate(!needs_acquiring_load(n));
6753
6754 ins_cost(4 * INSN_COST);
6755 format %{ "ldr $dst, $mem\t# class" %}
6756
6757 ins_encode(aarch64_enc_ldr(dst, mem));
6758
6759 ins_pipe(iload_reg_mem);
6760 %}
6761
6762 // Load Narrow Klass Pointer
6763 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6764 %{
6765 match(Set dst (LoadNKlass mem));
6766 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6767
6768 ins_cost(4 * INSN_COST);
6769 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6770
6771 ins_encode(aarch64_enc_ldrw(dst, mem));
6772
6773 ins_pipe(iload_reg_mem);
6774 %}
6775
6776 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6777 %{
6778 match(Set dst (LoadNKlass mem));
6779 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6780
6781 ins_cost(4 * INSN_COST);
6782 format %{
6783 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6784 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6785 %}
6786 ins_encode %{
6787 // inlined aarch64_enc_ldrw
6788 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6789 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6790 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6791 %}
6792 ins_pipe(iload_reg_mem);
6793 %}
6794
6795 // Load Float
6796 instruct loadF(vRegF dst, memory4 mem)
6797 %{
6798 match(Set dst (LoadF mem));
6799 predicate(!needs_acquiring_load(n));
6800
6801 ins_cost(4 * INSN_COST);
6802 format %{ "ldrs $dst, $mem\t# float" %}
6803
6804 ins_encode( aarch64_enc_ldrs(dst, mem) );
6805
6806 ins_pipe(pipe_class_memory);
6807 %}
6808
6809 // Load Double
6810 instruct loadD(vRegD dst, memory8 mem)
6811 %{
6812 match(Set dst (LoadD mem));
6813 predicate(!needs_acquiring_load(n));
6814
6815 ins_cost(4 * INSN_COST);
6816 format %{ "ldrd $dst, $mem\t# double" %}
6817
6818 ins_encode( aarch64_enc_ldrd(dst, mem) );
6819
6820 ins_pipe(pipe_class_memory);
6821 %}
6822
6823
6824 // Load Int Constant
6825 instruct loadConI(iRegINoSp dst, immI src)
6826 %{
6827 match(Set dst src);
6828
6829 ins_cost(INSN_COST);
6830 format %{ "mov $dst, $src\t# int" %}
6831
6832 ins_encode( aarch64_enc_movw_imm(dst, src) );
6833
6834 ins_pipe(ialu_imm);
6835 %}
6836
6837 // Load Long Constant
6838 instruct loadConL(iRegLNoSp dst, immL src)
6839 %{
6840 match(Set dst src);
6841
6842 ins_cost(INSN_COST);
6843 format %{ "mov $dst, $src\t# long" %}
6844
6845 ins_encode( aarch64_enc_mov_imm(dst, src) );
6846
6847 ins_pipe(ialu_imm);
6848 %}
6849
6850 // Load Pointer Constant
6851
6852 instruct loadConP(iRegPNoSp dst, immP con)
6853 %{
6854 match(Set dst con);
6855
6856 ins_cost(INSN_COST * 4);
6857 format %{
6858 "mov $dst, $con\t# ptr\n\t"
6859 %}
6860
6861 ins_encode(aarch64_enc_mov_p(dst, con));
6862
6863 ins_pipe(ialu_imm);
6864 %}
6865
6866 // Load Null Pointer Constant
6867
6868 instruct loadConP0(iRegPNoSp dst, immP0 con)
6869 %{
6870 match(Set dst con);
6871
6872 ins_cost(INSN_COST);
6873 format %{ "mov $dst, $con\t# nullptr ptr" %}
6874
6875 ins_encode(aarch64_enc_mov_p0(dst, con));
6876
6877 ins_pipe(ialu_imm);
6878 %}
6879
6880 // Load Pointer Constant One
6881
6882 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6883 %{
6884 match(Set dst con);
6885
6886 ins_cost(INSN_COST);
6887 format %{ "mov $dst, $con\t# nullptr ptr" %}
6888
6889 ins_encode(aarch64_enc_mov_p1(dst, con));
6890
6891 ins_pipe(ialu_imm);
6892 %}
6893
6894 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6895 %{
6896 match(Set dst con);
6897
6898 ins_cost(INSN_COST);
6899 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6900
6901 ins_encode %{
6902 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6903 %}
6904
6905 ins_pipe(ialu_imm);
6906 %}
6907
6908 // Load Narrow Pointer Constant
6909
6910 instruct loadConN(iRegNNoSp dst, immN con)
6911 %{
6912 match(Set dst con);
6913
6914 ins_cost(INSN_COST * 4);
6915 format %{ "mov $dst, $con\t# compressed ptr" %}
6916
6917 ins_encode(aarch64_enc_mov_n(dst, con));
6918
6919 ins_pipe(ialu_imm);
6920 %}
6921
6922 // Load Narrow Null Pointer Constant
6923
6924 instruct loadConN0(iRegNNoSp dst, immN0 con)
6925 %{
6926 match(Set dst con);
6927
6928 ins_cost(INSN_COST);
6929 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6930
6931 ins_encode(aarch64_enc_mov_n0(dst, con));
6932
6933 ins_pipe(ialu_imm);
6934 %}
6935
6936 // Load Narrow Klass Constant
6937
6938 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6939 %{
6940 match(Set dst con);
6941
6942 ins_cost(INSN_COST);
6943 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6944
6945 ins_encode(aarch64_enc_mov_nk(dst, con));
6946
6947 ins_pipe(ialu_imm);
6948 %}
6949
6950 // Load Packed Float Constant
6951
6952 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6953 match(Set dst con);
6954 ins_cost(INSN_COST * 4);
6955 format %{ "fmovs $dst, $con"%}
6956 ins_encode %{
6957 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6958 %}
6959
6960 ins_pipe(fp_imm_s);
6961 %}
6962
6963 // Load Float Constant
6964
6965 instruct loadConF(vRegF dst, immF con) %{
6966 match(Set dst con);
6967
6968 ins_cost(INSN_COST * 4);
6969
6970 format %{
6971 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6972 %}
6973
6974 ins_encode %{
6975 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6976 %}
6977
6978 ins_pipe(fp_load_constant_s);
6979 %}
6980
6981 // Load Packed Double Constant
6982
6983 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6984 match(Set dst con);
6985 ins_cost(INSN_COST);
6986 format %{ "fmovd $dst, $con"%}
6987 ins_encode %{
6988 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6989 %}
6990
6991 ins_pipe(fp_imm_d);
6992 %}
6993
6994 // Load Double Constant
6995
6996 instruct loadConD(vRegD dst, immD con) %{
6997 match(Set dst con);
6998
6999 ins_cost(INSN_COST * 5);
7000 format %{
7001 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7002 %}
7003
7004 ins_encode %{
7005 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7006 %}
7007
7008 ins_pipe(fp_load_constant_d);
7009 %}
7010
7011 // Load Half Float Constant
7012 instruct loadConH(vRegF dst, immH con) %{
7013 match(Set dst con);
7014 format %{ "mov rscratch1, $con\n\t"
7015 "fmov $dst, rscratch1"
7016 %}
7017 ins_encode %{
7018 __ movw(rscratch1, (uint32_t)$con$$constant);
7019 __ fmovs($dst$$FloatRegister, rscratch1);
7020 %}
7021 ins_pipe(pipe_class_default);
7022 %}
7023
7024 // Store Instructions
7025
7026 // Store Byte
7027 instruct storeB(iRegIorL2I src, memory1 mem)
7028 %{
7029 match(Set mem (StoreB mem src));
7030 predicate(!needs_releasing_store(n));
7031
7032 ins_cost(INSN_COST);
7033 format %{ "strb $src, $mem\t# byte" %}
7034
7035 ins_encode(aarch64_enc_strb(src, mem));
7036
7037 ins_pipe(istore_reg_mem);
7038 %}
7039
7040
7041 instruct storeimmB0(immI0 zero, memory1 mem)
7042 %{
7043 match(Set mem (StoreB mem zero));
7044 predicate(!needs_releasing_store(n));
7045
7046 ins_cost(INSN_COST);
7047 format %{ "strb rscractch2, $mem\t# byte" %}
7048
7049 ins_encode(aarch64_enc_strb0(mem));
7050
7051 ins_pipe(istore_mem);
7052 %}
7053
7054 // Store Char/Short
7055 instruct storeC(iRegIorL2I src, memory2 mem)
7056 %{
7057 match(Set mem (StoreC mem src));
7058 predicate(!needs_releasing_store(n));
7059
7060 ins_cost(INSN_COST);
7061 format %{ "strh $src, $mem\t# short" %}
7062
7063 ins_encode(aarch64_enc_strh(src, mem));
7064
7065 ins_pipe(istore_reg_mem);
7066 %}
7067
7068 instruct storeimmC0(immI0 zero, memory2 mem)
7069 %{
7070 match(Set mem (StoreC mem zero));
7071 predicate(!needs_releasing_store(n));
7072
7073 ins_cost(INSN_COST);
7074 format %{ "strh zr, $mem\t# short" %}
7075
7076 ins_encode(aarch64_enc_strh0(mem));
7077
7078 ins_pipe(istore_mem);
7079 %}
7080
7081 // Store Integer
7082
7083 instruct storeI(iRegIorL2I src, memory4 mem)
7084 %{
7085 match(Set mem(StoreI mem src));
7086 predicate(!needs_releasing_store(n));
7087
7088 ins_cost(INSN_COST);
7089 format %{ "strw $src, $mem\t# int" %}
7090
7091 ins_encode(aarch64_enc_strw(src, mem));
7092
7093 ins_pipe(istore_reg_mem);
7094 %}
7095
7096 instruct storeimmI0(immI0 zero, memory4 mem)
7097 %{
7098 match(Set mem(StoreI mem zero));
7099 predicate(!needs_releasing_store(n));
7100
7101 ins_cost(INSN_COST);
7102 format %{ "strw zr, $mem\t# int" %}
7103
7104 ins_encode(aarch64_enc_strw0(mem));
7105
7106 ins_pipe(istore_mem);
7107 %}
7108
7109 // Store Long (64 bit signed)
7110 instruct storeL(iRegL src, memory8 mem)
7111 %{
7112 match(Set mem (StoreL mem src));
7113 predicate(!needs_releasing_store(n));
7114
7115 ins_cost(INSN_COST);
7116 format %{ "str $src, $mem\t# int" %}
7117
7118 ins_encode(aarch64_enc_str(src, mem));
7119
7120 ins_pipe(istore_reg_mem);
7121 %}
7122
7123 // Store Long (64 bit signed)
7124 instruct storeimmL0(immL0 zero, memory8 mem)
7125 %{
7126 match(Set mem (StoreL mem zero));
7127 predicate(!needs_releasing_store(n));
7128
7129 ins_cost(INSN_COST);
7130 format %{ "str zr, $mem\t# int" %}
7131
7132 ins_encode(aarch64_enc_str0(mem));
7133
7134 ins_pipe(istore_mem);
7135 %}
7136
7137 // Store Pointer
7138 instruct storeP(iRegP src, memory8 mem)
7139 %{
7140 match(Set mem (StoreP mem src));
7141 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7142
7143 ins_cost(INSN_COST);
7144 format %{ "str $src, $mem\t# ptr" %}
7145
7146 ins_encode(aarch64_enc_str(src, mem));
7147
7148 ins_pipe(istore_reg_mem);
7149 %}
7150
7151 // Store Pointer
7152 instruct storeimmP0(immP0 zero, memory8 mem)
7153 %{
7154 match(Set mem (StoreP mem zero));
7155 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7156
7157 ins_cost(INSN_COST);
7158 format %{ "str zr, $mem\t# ptr" %}
7159
7160 ins_encode(aarch64_enc_str0(mem));
7161
7162 ins_pipe(istore_mem);
7163 %}
7164
7165 // Store Compressed Pointer
7166 instruct storeN(iRegN src, memory4 mem)
7167 %{
7168 match(Set mem (StoreN mem src));
7169 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7170
7171 ins_cost(INSN_COST);
7172 format %{ "strw $src, $mem\t# compressed ptr" %}
7173
7174 ins_encode(aarch64_enc_strw(src, mem));
7175
7176 ins_pipe(istore_reg_mem);
7177 %}
7178
7179 instruct storeImmN0(immN0 zero, memory4 mem)
7180 %{
7181 match(Set mem (StoreN mem zero));
7182 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7183
7184 ins_cost(INSN_COST);
7185 format %{ "strw zr, $mem\t# compressed ptr" %}
7186
7187 ins_encode(aarch64_enc_strw0(mem));
7188
7189 ins_pipe(istore_mem);
7190 %}
7191
7192 // Store Float
7193 instruct storeF(vRegF src, memory4 mem)
7194 %{
7195 match(Set mem (StoreF mem src));
7196 predicate(!needs_releasing_store(n));
7197
7198 ins_cost(INSN_COST);
7199 format %{ "strs $src, $mem\t# float" %}
7200
7201 ins_encode( aarch64_enc_strs(src, mem) );
7202
7203 ins_pipe(pipe_class_memory);
7204 %}
7205
7206 // TODO
7207 // implement storeImmF0 and storeFImmPacked
7208
7209 // Store Double
7210 instruct storeD(vRegD src, memory8 mem)
7211 %{
7212 match(Set mem (StoreD mem src));
7213 predicate(!needs_releasing_store(n));
7214
7215 ins_cost(INSN_COST);
7216 format %{ "strd $src, $mem\t# double" %}
7217
7218 ins_encode( aarch64_enc_strd(src, mem) );
7219
7220 ins_pipe(pipe_class_memory);
7221 %}
7222
7223 // Store Compressed Klass Pointer
7224 instruct storeNKlass(iRegN src, memory4 mem)
7225 %{
7226 predicate(!needs_releasing_store(n));
7227 match(Set mem (StoreNKlass mem src));
7228
7229 ins_cost(INSN_COST);
7230 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7231
7232 ins_encode(aarch64_enc_strw(src, mem));
7233
7234 ins_pipe(istore_reg_mem);
7235 %}
7236
7237 // TODO
7238 // implement storeImmD0 and storeDImmPacked
7239
7240 // prefetch instructions
7241 // Must be safe to execute with invalid address (cannot fault).
7242
7243 instruct prefetchalloc( memory8 mem ) %{
7244 match(PrefetchAllocation mem);
7245
7246 ins_cost(INSN_COST);
7247 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7248
7249 ins_encode( aarch64_enc_prefetchw(mem) );
7250
7251 ins_pipe(iload_prefetch);
7252 %}
7253
7254 // ---------------- volatile loads and stores ----------------
7255
7256 // Load Byte (8 bit signed)
7257 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7258 %{
7259 match(Set dst (LoadB mem));
7260
7261 ins_cost(VOLATILE_REF_COST);
7262 format %{ "ldarsb $dst, $mem\t# byte" %}
7263
7264 ins_encode(aarch64_enc_ldarsb(dst, mem));
7265
7266 ins_pipe(pipe_serial);
7267 %}
7268
7269 // Load Byte (8 bit signed) into long
7270 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7271 %{
7272 match(Set dst (ConvI2L (LoadB mem)));
7273
7274 ins_cost(VOLATILE_REF_COST);
7275 format %{ "ldarsb $dst, $mem\t# byte" %}
7276
7277 ins_encode(aarch64_enc_ldarsb(dst, mem));
7278
7279 ins_pipe(pipe_serial);
7280 %}
7281
7282 // Load Byte (8 bit unsigned)
7283 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7284 %{
7285 match(Set dst (LoadUB mem));
7286
7287 ins_cost(VOLATILE_REF_COST);
7288 format %{ "ldarb $dst, $mem\t# byte" %}
7289
7290 ins_encode(aarch64_enc_ldarb(dst, mem));
7291
7292 ins_pipe(pipe_serial);
7293 %}
7294
7295 // Load Byte (8 bit unsigned) into long
7296 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7297 %{
7298 match(Set dst (ConvI2L (LoadUB mem)));
7299
7300 ins_cost(VOLATILE_REF_COST);
7301 format %{ "ldarb $dst, $mem\t# byte" %}
7302
7303 ins_encode(aarch64_enc_ldarb(dst, mem));
7304
7305 ins_pipe(pipe_serial);
7306 %}
7307
7308 // Load Short (16 bit signed)
7309 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7310 %{
7311 match(Set dst (LoadS mem));
7312
7313 ins_cost(VOLATILE_REF_COST);
7314 format %{ "ldarshw $dst, $mem\t# short" %}
7315
7316 ins_encode(aarch64_enc_ldarshw(dst, mem));
7317
7318 ins_pipe(pipe_serial);
7319 %}
7320
7321 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7322 %{
7323 match(Set dst (LoadUS mem));
7324
7325 ins_cost(VOLATILE_REF_COST);
7326 format %{ "ldarhw $dst, $mem\t# short" %}
7327
7328 ins_encode(aarch64_enc_ldarhw(dst, mem));
7329
7330 ins_pipe(pipe_serial);
7331 %}
7332
7333 // Load Short/Char (16 bit unsigned) into long
7334 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7335 %{
7336 match(Set dst (ConvI2L (LoadUS mem)));
7337
7338 ins_cost(VOLATILE_REF_COST);
7339 format %{ "ldarh $dst, $mem\t# short" %}
7340
7341 ins_encode(aarch64_enc_ldarh(dst, mem));
7342
7343 ins_pipe(pipe_serial);
7344 %}
7345
7346 // Load Short/Char (16 bit signed) into long
7347 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7348 %{
7349 match(Set dst (ConvI2L (LoadS mem)));
7350
7351 ins_cost(VOLATILE_REF_COST);
7352 format %{ "ldarh $dst, $mem\t# short" %}
7353
7354 ins_encode(aarch64_enc_ldarsh(dst, mem));
7355
7356 ins_pipe(pipe_serial);
7357 %}
7358
7359 // Load Integer (32 bit signed)
7360 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7361 %{
7362 match(Set dst (LoadI mem));
7363
7364 ins_cost(VOLATILE_REF_COST);
7365 format %{ "ldarw $dst, $mem\t# int" %}
7366
7367 ins_encode(aarch64_enc_ldarw(dst, mem));
7368
7369 ins_pipe(pipe_serial);
7370 %}
7371
7372 // Load Integer (32 bit unsigned) into long
7373 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7374 %{
7375 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7376
7377 ins_cost(VOLATILE_REF_COST);
7378 format %{ "ldarw $dst, $mem\t# int" %}
7379
7380 ins_encode(aarch64_enc_ldarw(dst, mem));
7381
7382 ins_pipe(pipe_serial);
7383 %}
7384
7385 // Load Long (64 bit signed)
7386 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7387 %{
7388 match(Set dst (LoadL mem));
7389
7390 ins_cost(VOLATILE_REF_COST);
7391 format %{ "ldar $dst, $mem\t# int" %}
7392
7393 ins_encode(aarch64_enc_ldar(dst, mem));
7394
7395 ins_pipe(pipe_serial);
7396 %}
7397
7398 // Load Pointer
7399 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7400 %{
7401 match(Set dst (LoadP mem));
7402 predicate(n->as_Load()->barrier_data() == 0);
7403
7404 ins_cost(VOLATILE_REF_COST);
7405 format %{ "ldar $dst, $mem\t# ptr" %}
7406
7407 ins_encode(aarch64_enc_ldar(dst, mem));
7408
7409 ins_pipe(pipe_serial);
7410 %}
7411
7412 // Load Compressed Pointer
7413 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7414 %{
7415 match(Set dst (LoadN mem));
7416 predicate(n->as_Load()->barrier_data() == 0);
7417
7418 ins_cost(VOLATILE_REF_COST);
7419 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7420
7421 ins_encode(aarch64_enc_ldarw(dst, mem));
7422
7423 ins_pipe(pipe_serial);
7424 %}
7425
7426 // Load Float
7427 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7428 %{
7429 match(Set dst (LoadF mem));
7430
7431 ins_cost(VOLATILE_REF_COST);
7432 format %{ "ldars $dst, $mem\t# float" %}
7433
7434 ins_encode( aarch64_enc_fldars(dst, mem) );
7435
7436 ins_pipe(pipe_serial);
7437 %}
7438
7439 // Load Double
7440 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7441 %{
7442 match(Set dst (LoadD mem));
7443
7444 ins_cost(VOLATILE_REF_COST);
7445 format %{ "ldard $dst, $mem\t# double" %}
7446
7447 ins_encode( aarch64_enc_fldard(dst, mem) );
7448
7449 ins_pipe(pipe_serial);
7450 %}
7451
7452 // Store Byte
7453 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7454 %{
7455 match(Set mem (StoreB mem src));
7456
7457 ins_cost(VOLATILE_REF_COST);
7458 format %{ "stlrb $src, $mem\t# byte" %}
7459
7460 ins_encode(aarch64_enc_stlrb(src, mem));
7461
7462 ins_pipe(pipe_class_memory);
7463 %}
7464
7465 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7466 %{
7467 match(Set mem (StoreB mem zero));
7468
7469 ins_cost(VOLATILE_REF_COST);
7470 format %{ "stlrb zr, $mem\t# byte" %}
7471
7472 ins_encode(aarch64_enc_stlrb0(mem));
7473
7474 ins_pipe(pipe_class_memory);
7475 %}
7476
7477 // Store Char/Short
7478 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7479 %{
7480 match(Set mem (StoreC mem src));
7481
7482 ins_cost(VOLATILE_REF_COST);
7483 format %{ "stlrh $src, $mem\t# short" %}
7484
7485 ins_encode(aarch64_enc_stlrh(src, mem));
7486
7487 ins_pipe(pipe_class_memory);
7488 %}
7489
7490 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7491 %{
7492 match(Set mem (StoreC mem zero));
7493
7494 ins_cost(VOLATILE_REF_COST);
7495 format %{ "stlrh zr, $mem\t# short" %}
7496
7497 ins_encode(aarch64_enc_stlrh0(mem));
7498
7499 ins_pipe(pipe_class_memory);
7500 %}
7501
7502 // Store Integer
7503
7504 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7505 %{
7506 match(Set mem(StoreI mem src));
7507
7508 ins_cost(VOLATILE_REF_COST);
7509 format %{ "stlrw $src, $mem\t# int" %}
7510
7511 ins_encode(aarch64_enc_stlrw(src, mem));
7512
7513 ins_pipe(pipe_class_memory);
7514 %}
7515
7516 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7517 %{
7518 match(Set mem(StoreI mem zero));
7519
7520 ins_cost(VOLATILE_REF_COST);
7521 format %{ "stlrw zr, $mem\t# int" %}
7522
7523 ins_encode(aarch64_enc_stlrw0(mem));
7524
7525 ins_pipe(pipe_class_memory);
7526 %}
7527
7528 // Store Long (64 bit signed)
7529 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7530 %{
7531 match(Set mem (StoreL mem src));
7532
7533 ins_cost(VOLATILE_REF_COST);
7534 format %{ "stlr $src, $mem\t# int" %}
7535
7536 ins_encode(aarch64_enc_stlr(src, mem));
7537
7538 ins_pipe(pipe_class_memory);
7539 %}
7540
7541 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7542 %{
7543 match(Set mem (StoreL mem zero));
7544
7545 ins_cost(VOLATILE_REF_COST);
7546 format %{ "stlr zr, $mem\t# int" %}
7547
7548 ins_encode(aarch64_enc_stlr0(mem));
7549
7550 ins_pipe(pipe_class_memory);
7551 %}
7552
7553 // Store Pointer
7554 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7555 %{
7556 match(Set mem (StoreP mem src));
7557 predicate(n->as_Store()->barrier_data() == 0);
7558
7559 ins_cost(VOLATILE_REF_COST);
7560 format %{ "stlr $src, $mem\t# ptr" %}
7561
7562 ins_encode(aarch64_enc_stlr(src, mem));
7563
7564 ins_pipe(pipe_class_memory);
7565 %}
7566
7567 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7568 %{
7569 match(Set mem (StoreP mem zero));
7570 predicate(n->as_Store()->barrier_data() == 0);
7571
7572 ins_cost(VOLATILE_REF_COST);
7573 format %{ "stlr zr, $mem\t# ptr" %}
7574
7575 ins_encode(aarch64_enc_stlr0(mem));
7576
7577 ins_pipe(pipe_class_memory);
7578 %}
7579
7580 // Store Compressed Pointer
7581 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7582 %{
7583 match(Set mem (StoreN mem src));
7584 predicate(n->as_Store()->barrier_data() == 0);
7585
7586 ins_cost(VOLATILE_REF_COST);
7587 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7588
7589 ins_encode(aarch64_enc_stlrw(src, mem));
7590
7591 ins_pipe(pipe_class_memory);
7592 %}
7593
7594 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7595 %{
7596 match(Set mem (StoreN mem zero));
7597 predicate(n->as_Store()->barrier_data() == 0);
7598
7599 ins_cost(VOLATILE_REF_COST);
7600 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7601
7602 ins_encode(aarch64_enc_stlrw0(mem));
7603
7604 ins_pipe(pipe_class_memory);
7605 %}
7606
7607 // Store Float
7608 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7609 %{
7610 match(Set mem (StoreF mem src));
7611
7612 ins_cost(VOLATILE_REF_COST);
7613 format %{ "stlrs $src, $mem\t# float" %}
7614
7615 ins_encode( aarch64_enc_fstlrs(src, mem) );
7616
7617 ins_pipe(pipe_class_memory);
7618 %}
7619
7620 // TODO
7621 // implement storeImmF0 and storeFImmPacked
7622
7623 // Store Double
7624 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7625 %{
7626 match(Set mem (StoreD mem src));
7627
7628 ins_cost(VOLATILE_REF_COST);
7629 format %{ "stlrd $src, $mem\t# double" %}
7630
7631 ins_encode( aarch64_enc_fstlrd(src, mem) );
7632
7633 ins_pipe(pipe_class_memory);
7634 %}
7635
7636 // ---------------- end of volatile loads and stores ----------------
7637
7638 instruct cacheWB(indirect addr)
7639 %{
7640 predicate(VM_Version::supports_data_cache_line_flush());
7641 match(CacheWB addr);
7642
7643 ins_cost(100);
7644 format %{"cache wb $addr" %}
7645 ins_encode %{
7646 assert($addr->index_position() < 0, "should be");
7647 assert($addr$$disp == 0, "should be");
7648 __ cache_wb(Address($addr$$base$$Register, 0));
7649 %}
7650 ins_pipe(pipe_slow); // XXX
7651 %}
7652
7653 instruct cacheWBPreSync()
7654 %{
7655 predicate(VM_Version::supports_data_cache_line_flush());
7656 match(CacheWBPreSync);
7657
7658 ins_cost(100);
7659 format %{"cache wb presync" %}
7660 ins_encode %{
7661 __ cache_wbsync(true);
7662 %}
7663 ins_pipe(pipe_slow); // XXX
7664 %}
7665
7666 instruct cacheWBPostSync()
7667 %{
7668 predicate(VM_Version::supports_data_cache_line_flush());
7669 match(CacheWBPostSync);
7670
7671 ins_cost(100);
7672 format %{"cache wb postsync" %}
7673 ins_encode %{
7674 __ cache_wbsync(false);
7675 %}
7676 ins_pipe(pipe_slow); // XXX
7677 %}
7678
7679 // ============================================================================
7680 // BSWAP Instructions
7681
7682 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7683 match(Set dst (ReverseBytesI src));
7684
7685 ins_cost(INSN_COST);
7686 format %{ "revw $dst, $src" %}
7687
7688 ins_encode %{
7689 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7690 %}
7691
7692 ins_pipe(ialu_reg);
7693 %}
7694
7695 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7696 match(Set dst (ReverseBytesL src));
7697
7698 ins_cost(INSN_COST);
7699 format %{ "rev $dst, $src" %}
7700
7701 ins_encode %{
7702 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7703 %}
7704
7705 ins_pipe(ialu_reg);
7706 %}
7707
7708 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7709 match(Set dst (ReverseBytesUS src));
7710
7711 ins_cost(INSN_COST);
7712 format %{ "rev16w $dst, $src\t# $dst -> unsigned short" %}
7713
7714 ins_encode %{
7715 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7716 __ narrow_subword_type(as_Register($dst$$reg), T_CHAR);
7717 %}
7718
7719 ins_pipe(ialu_reg);
7720 %}
7721
7722 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7723 match(Set dst (ReverseBytesS src));
7724
7725 ins_cost(INSN_COST);
7726 format %{ "rev16w $dst, $src\n\t"
7727 "sbfmw $dst, $dst, #0, #15" %}
7728
7729 ins_encode %{
7730 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7731 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7732 %}
7733
7734 ins_pipe(ialu_reg);
7735 %}
7736
7737 // ============================================================================
7738 // Zero Count Instructions
7739
7740 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7741 match(Set dst (CountLeadingZerosI src));
7742
7743 ins_cost(INSN_COST);
7744 format %{ "clzw $dst, $src" %}
7745 ins_encode %{
7746 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7747 %}
7748
7749 ins_pipe(ialu_reg);
7750 %}
7751
7752 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7753 match(Set dst (CountLeadingZerosL src));
7754
7755 ins_cost(INSN_COST);
7756 format %{ "clz $dst, $src" %}
7757 ins_encode %{
7758 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7759 %}
7760
7761 ins_pipe(ialu_reg);
7762 %}
7763
7764 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7765 match(Set dst (CountTrailingZerosI src));
7766
7767 ins_cost(INSN_COST * 2);
7768 format %{ "rbitw $dst, $src\n\t"
7769 "clzw $dst, $dst" %}
7770 ins_encode %{
7771 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7772 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7773 %}
7774
7775 ins_pipe(ialu_reg);
7776 %}
7777
7778 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7779 match(Set dst (CountTrailingZerosL src));
7780
7781 ins_cost(INSN_COST * 2);
7782 format %{ "rbit $dst, $src\n\t"
7783 "clz $dst, $dst" %}
7784 ins_encode %{
7785 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7786 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7787 %}
7788
7789 ins_pipe(ialu_reg);
7790 %}
7791
7792 //---------- Population Count Instructions -------------------------------------
7793 //
7794
7795 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7796 match(Set dst (PopCountI src));
7797 effect(TEMP tmp);
7798 ins_cost(INSN_COST * 13);
7799
7800 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7801 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7802 "addv $tmp, $tmp\t# vector (8B)\n\t"
7803 "mov $dst, $tmp\t# vector (1D)" %}
7804 ins_encode %{
7805 __ fmovs($tmp$$FloatRegister, $src$$Register);
7806 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7807 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7808 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7809 %}
7810
7811 ins_pipe(pipe_class_default);
7812 %}
7813
7814 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7815 match(Set dst (PopCountI (LoadI mem)));
7816 effect(TEMP tmp);
7817 ins_cost(INSN_COST * 13);
7818
7819 format %{ "ldrs $tmp, $mem\n\t"
7820 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7821 "addv $tmp, $tmp\t# vector (8B)\n\t"
7822 "mov $dst, $tmp\t# vector (1D)" %}
7823 ins_encode %{
7824 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7825 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7826 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7827 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7828 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7829 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7830 %}
7831
7832 ins_pipe(pipe_class_default);
7833 %}
7834
7835 // Note: Long.bitCount(long) returns an int.
7836 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7837 match(Set dst (PopCountL src));
7838 effect(TEMP tmp);
7839 ins_cost(INSN_COST * 13);
7840
7841 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7842 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7843 "addv $tmp, $tmp\t# vector (8B)\n\t"
7844 "mov $dst, $tmp\t# vector (1D)" %}
7845 ins_encode %{
7846 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7847 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7848 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7849 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7850 %}
7851
7852 ins_pipe(pipe_class_default);
7853 %}
7854
7855 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7856 match(Set dst (PopCountL (LoadL mem)));
7857 effect(TEMP tmp);
7858 ins_cost(INSN_COST * 13);
7859
7860 format %{ "ldrd $tmp, $mem\n\t"
7861 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7862 "addv $tmp, $tmp\t# vector (8B)\n\t"
7863 "mov $dst, $tmp\t# vector (1D)" %}
7864 ins_encode %{
7865 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7866 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7867 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7868 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7869 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7870 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7871 %}
7872
7873 ins_pipe(pipe_class_default);
7874 %}
7875
7876 // ============================================================================
7877 // VerifyVectorAlignment Instruction
7878
7879 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7880 match(Set addr (VerifyVectorAlignment addr mask));
7881 effect(KILL cr);
7882 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7883 ins_encode %{
7884 Label Lskip;
7885 // check if masked bits of addr are zero
7886 __ tst($addr$$Register, $mask$$constant);
7887 __ br(Assembler::EQ, Lskip);
7888 __ stop("verify_vector_alignment found a misaligned vector memory access");
7889 __ bind(Lskip);
7890 %}
7891 ins_pipe(pipe_slow);
7892 %}
7893
7894 // ============================================================================
7895 // MemBar Instruction
7896
7897 instruct load_fence() %{
7898 match(LoadFence);
7899 ins_cost(VOLATILE_REF_COST);
7900
7901 format %{ "load_fence" %}
7902
7903 ins_encode %{
7904 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7905 %}
7906 ins_pipe(pipe_serial);
7907 %}
7908
7909 instruct unnecessary_membar_acquire() %{
7910 predicate(unnecessary_acquire(n));
7911 match(MemBarAcquire);
7912 ins_cost(0);
7913
7914 format %{ "membar_acquire (elided)" %}
7915
7916 ins_encode %{
7917 __ block_comment("membar_acquire (elided)");
7918 %}
7919
7920 ins_pipe(pipe_class_empty);
7921 %}
7922
7923 instruct membar_acquire() %{
7924 match(MemBarAcquire);
7925 ins_cost(VOLATILE_REF_COST);
7926
7927 format %{ "membar_acquire\n\t"
7928 "dmb ishld" %}
7929
7930 ins_encode %{
7931 __ block_comment("membar_acquire");
7932 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7933 %}
7934
7935 ins_pipe(pipe_serial);
7936 %}
7937
7938
7939 instruct membar_acquire_lock() %{
7940 match(MemBarAcquireLock);
7941 ins_cost(VOLATILE_REF_COST);
7942
7943 format %{ "membar_acquire_lock (elided)" %}
7944
7945 ins_encode %{
7946 __ block_comment("membar_acquire_lock (elided)");
7947 %}
7948
7949 ins_pipe(pipe_serial);
7950 %}
7951
7952 instruct store_fence() %{
7953 match(StoreFence);
7954 ins_cost(VOLATILE_REF_COST);
7955
7956 format %{ "store_fence" %}
7957
7958 ins_encode %{
7959 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7960 %}
7961 ins_pipe(pipe_serial);
7962 %}
7963
7964 instruct unnecessary_membar_release() %{
7965 predicate(unnecessary_release(n));
7966 match(MemBarRelease);
7967 ins_cost(0);
7968
7969 format %{ "membar_release (elided)" %}
7970
7971 ins_encode %{
7972 __ block_comment("membar_release (elided)");
7973 %}
7974 ins_pipe(pipe_serial);
7975 %}
7976
7977 instruct membar_release() %{
7978 match(MemBarRelease);
7979 ins_cost(VOLATILE_REF_COST);
7980
7981 format %{ "membar_release\n\t"
7982 "dmb ishst\n\tdmb ishld" %}
7983
7984 ins_encode %{
7985 __ block_comment("membar_release");
7986 // These will be merged if AlwaysMergeDMB is enabled.
7987 __ membar(Assembler::StoreStore);
7988 __ membar(Assembler::LoadStore);
7989 %}
7990 ins_pipe(pipe_serial);
7991 %}
7992
7993 instruct membar_storestore() %{
7994 match(MemBarStoreStore);
7995 match(StoreStoreFence);
7996 ins_cost(VOLATILE_REF_COST);
7997
7998 format %{ "MEMBAR-store-store" %}
7999
8000 ins_encode %{
8001 __ membar(Assembler::StoreStore);
8002 %}
8003 ins_pipe(pipe_serial);
8004 %}
8005
8006 instruct membar_release_lock() %{
8007 match(MemBarReleaseLock);
8008 ins_cost(VOLATILE_REF_COST);
8009
8010 format %{ "membar_release_lock (elided)" %}
8011
8012 ins_encode %{
8013 __ block_comment("membar_release_lock (elided)");
8014 %}
8015
8016 ins_pipe(pipe_serial);
8017 %}
8018
8019 instruct membar_storeload() %{
8020 match(MemBarStoreLoad);
8021 ins_cost(VOLATILE_REF_COST*100);
8022
8023 format %{ "MEMBAR-store-load\n\t"
8024 "dmb ish" %}
8025
8026 ins_encode %{
8027 __ block_comment("membar_storeload");
8028 __ membar(Assembler::StoreLoad);
8029 %}
8030
8031 ins_pipe(pipe_serial);
8032 %}
8033
8034 instruct unnecessary_membar_volatile() %{
8035 predicate(unnecessary_volatile(n));
8036 match(MemBarVolatile);
8037 ins_cost(0);
8038
8039 format %{ "membar_volatile (elided)" %}
8040
8041 ins_encode %{
8042 __ block_comment("membar_volatile (elided)");
8043 %}
8044
8045 ins_pipe(pipe_serial);
8046 %}
8047
8048 instruct membar_volatile() %{
8049 match(MemBarVolatile);
8050 ins_cost(VOLATILE_REF_COST*100);
8051
8052 format %{ "membar_volatile\n\t"
8053 "dmb ish"%}
8054
8055 ins_encode %{
8056 __ block_comment("membar_volatile");
8057 __ membar(Assembler::StoreLoad);
8058 %}
8059
8060 ins_pipe(pipe_serial);
8061 %}
8062
8063 instruct membar_full() %{
8064 match(MemBarFull);
8065 ins_cost(VOLATILE_REF_COST*100);
8066
8067 format %{ "membar_full\n\t"
8068 "dmb ish" %}
8069 ins_encode %{
8070 __ block_comment("membar_full");
8071 __ membar(Assembler::AnyAny);
8072 %}
8073
8074 ins_pipe(pipe_serial);
8075 %}
8076
8077 // ============================================================================
8078 // Cast/Convert Instructions
8079
8080 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8081 match(Set dst (CastX2P src));
8082
8083 ins_cost(INSN_COST);
8084 format %{ "mov $dst, $src\t# long -> ptr" %}
8085
8086 ins_encode %{
8087 if ($dst$$reg != $src$$reg) {
8088 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8089 }
8090 %}
8091
8092 ins_pipe(ialu_reg);
8093 %}
8094
8095 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8096 match(Set dst (CastP2X src));
8097
8098 ins_cost(INSN_COST);
8099 format %{ "mov $dst, $src\t# ptr -> long" %}
8100
8101 ins_encode %{
8102 if ($dst$$reg != $src$$reg) {
8103 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8104 }
8105 %}
8106
8107 ins_pipe(ialu_reg);
8108 %}
8109
8110 // Convert oop into int for vectors alignment masking
8111 instruct convP2I(iRegINoSp dst, iRegP src) %{
8112 match(Set dst (ConvL2I (CastP2X src)));
8113
8114 ins_cost(INSN_COST);
8115 format %{ "movw $dst, $src\t# ptr -> int" %}
8116 ins_encode %{
8117 __ movw($dst$$Register, $src$$Register);
8118 %}
8119
8120 ins_pipe(ialu_reg);
8121 %}
8122
8123 // Convert compressed oop into int for vectors alignment masking
8124 // in case of 32bit oops (heap < 4Gb).
8125 instruct convN2I(iRegINoSp dst, iRegN src)
8126 %{
8127 predicate(CompressedOops::shift() == 0);
8128 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8129
8130 ins_cost(INSN_COST);
8131 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8132 ins_encode %{
8133 __ movw($dst$$Register, $src$$Register);
8134 %}
8135
8136 ins_pipe(ialu_reg);
8137 %}
8138
8139
8140 // Convert oop pointer into compressed form
8141 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8142 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8143 match(Set dst (EncodeP src));
8144 effect(KILL cr);
8145 ins_cost(INSN_COST * 3);
8146 format %{ "encode_heap_oop $dst, $src" %}
8147 ins_encode %{
8148 Register s = $src$$Register;
8149 Register d = $dst$$Register;
8150 __ encode_heap_oop(d, s);
8151 %}
8152 ins_pipe(ialu_reg);
8153 %}
8154
8155 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8156 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8157 match(Set dst (EncodeP src));
8158 ins_cost(INSN_COST * 3);
8159 format %{ "encode_heap_oop_not_null $dst, $src" %}
8160 ins_encode %{
8161 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8162 %}
8163 ins_pipe(ialu_reg);
8164 %}
8165
8166 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8167 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8168 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8169 match(Set dst (DecodeN src));
8170 ins_cost(INSN_COST * 3);
8171 format %{ "decode_heap_oop $dst, $src" %}
8172 ins_encode %{
8173 Register s = $src$$Register;
8174 Register d = $dst$$Register;
8175 __ decode_heap_oop(d, s);
8176 %}
8177 ins_pipe(ialu_reg);
8178 %}
8179
8180 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8181 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8182 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8183 match(Set dst (DecodeN src));
8184 ins_cost(INSN_COST * 3);
8185 format %{ "decode_heap_oop_not_null $dst, $src" %}
8186 ins_encode %{
8187 Register s = $src$$Register;
8188 Register d = $dst$$Register;
8189 __ decode_heap_oop_not_null(d, s);
8190 %}
8191 ins_pipe(ialu_reg);
8192 %}
8193
8194 // n.b. AArch64 implementations of encode_klass_not_null and
8195 // decode_klass_not_null do not modify the flags register so, unlike
8196 // Intel, we don't kill CR as a side effect here
8197
8198 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8199 match(Set dst (EncodePKlass src));
8200
8201 ins_cost(INSN_COST * 3);
8202 format %{ "encode_klass_not_null $dst,$src" %}
8203
8204 ins_encode %{
8205 Register src_reg = as_Register($src$$reg);
8206 Register dst_reg = as_Register($dst$$reg);
8207 __ encode_klass_not_null(dst_reg, src_reg);
8208 %}
8209
8210 ins_pipe(ialu_reg);
8211 %}
8212
8213 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8214 match(Set dst (DecodeNKlass src));
8215
8216 ins_cost(INSN_COST * 3);
8217 format %{ "decode_klass_not_null $dst,$src" %}
8218
8219 ins_encode %{
8220 Register src_reg = as_Register($src$$reg);
8221 Register dst_reg = as_Register($dst$$reg);
8222 if (dst_reg != src_reg) {
8223 __ decode_klass_not_null(dst_reg, src_reg);
8224 } else {
8225 __ decode_klass_not_null(dst_reg);
8226 }
8227 %}
8228
8229 ins_pipe(ialu_reg);
8230 %}
8231
8232 instruct checkCastPP(iRegPNoSp dst)
8233 %{
8234 match(Set dst (CheckCastPP dst));
8235
8236 size(0);
8237 format %{ "# checkcastPP of $dst" %}
8238 ins_encode(/* empty encoding */);
8239 ins_pipe(pipe_class_empty);
8240 %}
8241
8242 instruct castPP(iRegPNoSp dst)
8243 %{
8244 match(Set dst (CastPP dst));
8245
8246 size(0);
8247 format %{ "# castPP of $dst" %}
8248 ins_encode(/* empty encoding */);
8249 ins_pipe(pipe_class_empty);
8250 %}
8251
8252 instruct castII(iRegI dst)
8253 %{
8254 predicate(VerifyConstraintCasts == 0);
8255 match(Set dst (CastII dst));
8256
8257 size(0);
8258 format %{ "# castII of $dst" %}
8259 ins_encode(/* empty encoding */);
8260 ins_cost(0);
8261 ins_pipe(pipe_class_empty);
8262 %}
8263
8264 instruct castII_checked(iRegI dst, rFlagsReg cr)
8265 %{
8266 predicate(VerifyConstraintCasts > 0);
8267 match(Set dst (CastII dst));
8268 effect(KILL cr);
8269
8270 format %{ "# castII_checked of $dst" %}
8271 ins_encode %{
8272 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8273 %}
8274 ins_pipe(pipe_slow);
8275 %}
8276
8277 instruct castLL(iRegL dst)
8278 %{
8279 predicate(VerifyConstraintCasts == 0);
8280 match(Set dst (CastLL dst));
8281
8282 size(0);
8283 format %{ "# castLL of $dst" %}
8284 ins_encode(/* empty encoding */);
8285 ins_cost(0);
8286 ins_pipe(pipe_class_empty);
8287 %}
8288
8289 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8290 %{
8291 predicate(VerifyConstraintCasts > 0);
8292 match(Set dst (CastLL dst));
8293 effect(KILL cr);
8294
8295 format %{ "# castLL_checked of $dst" %}
8296 ins_encode %{
8297 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8298 %}
8299 ins_pipe(pipe_slow);
8300 %}
8301
8302 instruct castHH(vRegF dst)
8303 %{
8304 match(Set dst (CastHH dst));
8305 size(0);
8306 format %{ "# castHH of $dst" %}
8307 ins_encode(/* empty encoding */);
8308 ins_cost(0);
8309 ins_pipe(pipe_class_empty);
8310 %}
8311
8312 instruct castFF(vRegF dst)
8313 %{
8314 match(Set dst (CastFF dst));
8315
8316 size(0);
8317 format %{ "# castFF of $dst" %}
8318 ins_encode(/* empty encoding */);
8319 ins_cost(0);
8320 ins_pipe(pipe_class_empty);
8321 %}
8322
8323 instruct castDD(vRegD dst)
8324 %{
8325 match(Set dst (CastDD dst));
8326
8327 size(0);
8328 format %{ "# castDD of $dst" %}
8329 ins_encode(/* empty encoding */);
8330 ins_cost(0);
8331 ins_pipe(pipe_class_empty);
8332 %}
8333
8334 instruct castVV(vReg dst)
8335 %{
8336 match(Set dst (CastVV dst));
8337
8338 size(0);
8339 format %{ "# castVV of $dst" %}
8340 ins_encode(/* empty encoding */);
8341 ins_cost(0);
8342 ins_pipe(pipe_class_empty);
8343 %}
8344
8345 instruct castVVMask(pRegGov dst)
8346 %{
8347 match(Set dst (CastVV dst));
8348
8349 size(0);
8350 format %{ "# castVV of $dst" %}
8351 ins_encode(/* empty encoding */);
8352 ins_cost(0);
8353 ins_pipe(pipe_class_empty);
8354 %}
8355
8356 // Manifest a CmpU result in an integer register.
8357 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8358 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8359 %{
8360 match(Set dst (CmpU3 src1 src2));
8361 effect(KILL flags);
8362
8363 ins_cost(INSN_COST * 3);
8364 format %{
8365 "cmpw $src1, $src2\n\t"
8366 "csetw $dst, ne\n\t"
8367 "cnegw $dst, lo\t# CmpU3(reg)"
8368 %}
8369 ins_encode %{
8370 __ cmpw($src1$$Register, $src2$$Register);
8371 __ csetw($dst$$Register, Assembler::NE);
8372 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8373 %}
8374
8375 ins_pipe(pipe_class_default);
8376 %}
8377
8378 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8379 %{
8380 match(Set dst (CmpU3 src1 src2));
8381 effect(KILL flags);
8382
8383 ins_cost(INSN_COST * 3);
8384 format %{
8385 "subsw zr, $src1, $src2\n\t"
8386 "csetw $dst, ne\n\t"
8387 "cnegw $dst, lo\t# CmpU3(imm)"
8388 %}
8389 ins_encode %{
8390 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8391 __ csetw($dst$$Register, Assembler::NE);
8392 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8393 %}
8394
8395 ins_pipe(pipe_class_default);
8396 %}
8397
8398 // Manifest a CmpUL result in an integer register.
8399 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8400 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8401 %{
8402 match(Set dst (CmpUL3 src1 src2));
8403 effect(KILL flags);
8404
8405 ins_cost(INSN_COST * 3);
8406 format %{
8407 "cmp $src1, $src2\n\t"
8408 "csetw $dst, ne\n\t"
8409 "cnegw $dst, lo\t# CmpUL3(reg)"
8410 %}
8411 ins_encode %{
8412 __ cmp($src1$$Register, $src2$$Register);
8413 __ csetw($dst$$Register, Assembler::NE);
8414 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8415 %}
8416
8417 ins_pipe(pipe_class_default);
8418 %}
8419
8420 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8421 %{
8422 match(Set dst (CmpUL3 src1 src2));
8423 effect(KILL flags);
8424
8425 ins_cost(INSN_COST * 3);
8426 format %{
8427 "subs zr, $src1, $src2\n\t"
8428 "csetw $dst, ne\n\t"
8429 "cnegw $dst, lo\t# CmpUL3(imm)"
8430 %}
8431 ins_encode %{
8432 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8433 __ csetw($dst$$Register, Assembler::NE);
8434 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8435 %}
8436
8437 ins_pipe(pipe_class_default);
8438 %}
8439
8440 // Manifest a CmpL result in an integer register.
8441 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8442 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8443 %{
8444 match(Set dst (CmpL3 src1 src2));
8445 effect(KILL flags);
8446
8447 ins_cost(INSN_COST * 3);
8448 format %{
8449 "cmp $src1, $src2\n\t"
8450 "csetw $dst, ne\n\t"
8451 "cnegw $dst, lt\t# CmpL3(reg)"
8452 %}
8453 ins_encode %{
8454 __ cmp($src1$$Register, $src2$$Register);
8455 __ csetw($dst$$Register, Assembler::NE);
8456 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8457 %}
8458
8459 ins_pipe(pipe_class_default);
8460 %}
8461
8462 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8463 %{
8464 match(Set dst (CmpL3 src1 src2));
8465 effect(KILL flags);
8466
8467 ins_cost(INSN_COST * 3);
8468 format %{
8469 "subs zr, $src1, $src2\n\t"
8470 "csetw $dst, ne\n\t"
8471 "cnegw $dst, lt\t# CmpL3(imm)"
8472 %}
8473 ins_encode %{
8474 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8475 __ csetw($dst$$Register, Assembler::NE);
8476 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8477 %}
8478
8479 ins_pipe(pipe_class_default);
8480 %}
8481
8482 // ============================================================================
8483 // Conditional Move Instructions
8484
8485 // n.b. we have identical rules for both a signed compare op (cmpOp)
8486 // and an unsigned compare op (cmpOpU). it would be nice if we could
8487 // define an op class which merged both inputs and use it to type the
8488 // argument to a single rule. unfortunatelyt his fails because the
8489 // opclass does not live up to the COND_INTER interface of its
8490 // component operands. When the generic code tries to negate the
8491 // operand it ends up running the generci Machoper::negate method
8492 // which throws a ShouldNotHappen. So, we have to provide two flavours
8493 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8494
8495 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8496 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8497
8498 ins_cost(INSN_COST * 2);
8499 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8500
8501 ins_encode %{
8502 __ cselw(as_Register($dst$$reg),
8503 as_Register($src2$$reg),
8504 as_Register($src1$$reg),
8505 (Assembler::Condition)$cmp$$cmpcode);
8506 %}
8507
8508 ins_pipe(icond_reg_reg);
8509 %}
8510
8511 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8512 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8513
8514 ins_cost(INSN_COST * 2);
8515 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8516
8517 ins_encode %{
8518 __ cselw(as_Register($dst$$reg),
8519 as_Register($src2$$reg),
8520 as_Register($src1$$reg),
8521 (Assembler::Condition)$cmp$$cmpcode);
8522 %}
8523
8524 ins_pipe(icond_reg_reg);
8525 %}
8526
8527 // special cases where one arg is zero
8528
8529 // n.b. this is selected in preference to the rule above because it
8530 // avoids loading constant 0 into a source register
8531
8532 // TODO
8533 // we ought only to be able to cull one of these variants as the ideal
8534 // transforms ought always to order the zero consistently (to left/right?)
8535
8536 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8537 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8538
8539 ins_cost(INSN_COST * 2);
8540 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8541
8542 ins_encode %{
8543 __ cselw(as_Register($dst$$reg),
8544 as_Register($src$$reg),
8545 zr,
8546 (Assembler::Condition)$cmp$$cmpcode);
8547 %}
8548
8549 ins_pipe(icond_reg);
8550 %}
8551
8552 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8553 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8554
8555 ins_cost(INSN_COST * 2);
8556 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8557
8558 ins_encode %{
8559 __ cselw(as_Register($dst$$reg),
8560 as_Register($src$$reg),
8561 zr,
8562 (Assembler::Condition)$cmp$$cmpcode);
8563 %}
8564
8565 ins_pipe(icond_reg);
8566 %}
8567
8568 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8569 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8570
8571 ins_cost(INSN_COST * 2);
8572 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8573
8574 ins_encode %{
8575 __ cselw(as_Register($dst$$reg),
8576 zr,
8577 as_Register($src$$reg),
8578 (Assembler::Condition)$cmp$$cmpcode);
8579 %}
8580
8581 ins_pipe(icond_reg);
8582 %}
8583
8584 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8585 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8586
8587 ins_cost(INSN_COST * 2);
8588 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8589
8590 ins_encode %{
8591 __ cselw(as_Register($dst$$reg),
8592 zr,
8593 as_Register($src$$reg),
8594 (Assembler::Condition)$cmp$$cmpcode);
8595 %}
8596
8597 ins_pipe(icond_reg);
8598 %}
8599
8600 // special case for creating a boolean 0 or 1
8601
8602 // n.b. this is selected in preference to the rule above because it
8603 // avoids loading constants 0 and 1 into a source register
8604
8605 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8606 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8607
8608 ins_cost(INSN_COST * 2);
8609 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8610
8611 ins_encode %{
8612 // equivalently
8613 // cset(as_Register($dst$$reg),
8614 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8615 __ csincw(as_Register($dst$$reg),
8616 zr,
8617 zr,
8618 (Assembler::Condition)$cmp$$cmpcode);
8619 %}
8620
8621 ins_pipe(icond_none);
8622 %}
8623
8624 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8625 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8626
8627 ins_cost(INSN_COST * 2);
8628 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8629
8630 ins_encode %{
8631 // equivalently
8632 // cset(as_Register($dst$$reg),
8633 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8634 __ csincw(as_Register($dst$$reg),
8635 zr,
8636 zr,
8637 (Assembler::Condition)$cmp$$cmpcode);
8638 %}
8639
8640 ins_pipe(icond_none);
8641 %}
8642
8643 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8644 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8645
8646 ins_cost(INSN_COST * 2);
8647 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8648
8649 ins_encode %{
8650 __ csel(as_Register($dst$$reg),
8651 as_Register($src2$$reg),
8652 as_Register($src1$$reg),
8653 (Assembler::Condition)$cmp$$cmpcode);
8654 %}
8655
8656 ins_pipe(icond_reg_reg);
8657 %}
8658
8659 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8660 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8661
8662 ins_cost(INSN_COST * 2);
8663 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8664
8665 ins_encode %{
8666 __ csel(as_Register($dst$$reg),
8667 as_Register($src2$$reg),
8668 as_Register($src1$$reg),
8669 (Assembler::Condition)$cmp$$cmpcode);
8670 %}
8671
8672 ins_pipe(icond_reg_reg);
8673 %}
8674
8675 // special cases where one arg is zero
8676
8677 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8678 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8679
8680 ins_cost(INSN_COST * 2);
8681 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8682
8683 ins_encode %{
8684 __ csel(as_Register($dst$$reg),
8685 zr,
8686 as_Register($src$$reg),
8687 (Assembler::Condition)$cmp$$cmpcode);
8688 %}
8689
8690 ins_pipe(icond_reg);
8691 %}
8692
8693 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8694 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8695
8696 ins_cost(INSN_COST * 2);
8697 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8698
8699 ins_encode %{
8700 __ csel(as_Register($dst$$reg),
8701 zr,
8702 as_Register($src$$reg),
8703 (Assembler::Condition)$cmp$$cmpcode);
8704 %}
8705
8706 ins_pipe(icond_reg);
8707 %}
8708
8709 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8710 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8711
8712 ins_cost(INSN_COST * 2);
8713 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8714
8715 ins_encode %{
8716 __ csel(as_Register($dst$$reg),
8717 as_Register($src$$reg),
8718 zr,
8719 (Assembler::Condition)$cmp$$cmpcode);
8720 %}
8721
8722 ins_pipe(icond_reg);
8723 %}
8724
8725 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8726 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8727
8728 ins_cost(INSN_COST * 2);
8729 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8730
8731 ins_encode %{
8732 __ csel(as_Register($dst$$reg),
8733 as_Register($src$$reg),
8734 zr,
8735 (Assembler::Condition)$cmp$$cmpcode);
8736 %}
8737
8738 ins_pipe(icond_reg);
8739 %}
8740
8741 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8742 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8743
8744 ins_cost(INSN_COST * 2);
8745 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8746
8747 ins_encode %{
8748 __ csel(as_Register($dst$$reg),
8749 as_Register($src2$$reg),
8750 as_Register($src1$$reg),
8751 (Assembler::Condition)$cmp$$cmpcode);
8752 %}
8753
8754 ins_pipe(icond_reg_reg);
8755 %}
8756
8757 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8758 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8759
8760 ins_cost(INSN_COST * 2);
8761 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8762
8763 ins_encode %{
8764 __ csel(as_Register($dst$$reg),
8765 as_Register($src2$$reg),
8766 as_Register($src1$$reg),
8767 (Assembler::Condition)$cmp$$cmpcode);
8768 %}
8769
8770 ins_pipe(icond_reg_reg);
8771 %}
8772
8773 // special cases where one arg is zero
8774
8775 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8776 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8777
8778 ins_cost(INSN_COST * 2);
8779 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8780
8781 ins_encode %{
8782 __ csel(as_Register($dst$$reg),
8783 zr,
8784 as_Register($src$$reg),
8785 (Assembler::Condition)$cmp$$cmpcode);
8786 %}
8787
8788 ins_pipe(icond_reg);
8789 %}
8790
8791 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8792 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8793
8794 ins_cost(INSN_COST * 2);
8795 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8796
8797 ins_encode %{
8798 __ csel(as_Register($dst$$reg),
8799 zr,
8800 as_Register($src$$reg),
8801 (Assembler::Condition)$cmp$$cmpcode);
8802 %}
8803
8804 ins_pipe(icond_reg);
8805 %}
8806
8807 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8808 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8809
8810 ins_cost(INSN_COST * 2);
8811 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8812
8813 ins_encode %{
8814 __ csel(as_Register($dst$$reg),
8815 as_Register($src$$reg),
8816 zr,
8817 (Assembler::Condition)$cmp$$cmpcode);
8818 %}
8819
8820 ins_pipe(icond_reg);
8821 %}
8822
8823 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8824 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8825
8826 ins_cost(INSN_COST * 2);
8827 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8828
8829 ins_encode %{
8830 __ csel(as_Register($dst$$reg),
8831 as_Register($src$$reg),
8832 zr,
8833 (Assembler::Condition)$cmp$$cmpcode);
8834 %}
8835
8836 ins_pipe(icond_reg);
8837 %}
8838
8839 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8840 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8841
8842 ins_cost(INSN_COST * 2);
8843 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8844
8845 ins_encode %{
8846 __ cselw(as_Register($dst$$reg),
8847 as_Register($src2$$reg),
8848 as_Register($src1$$reg),
8849 (Assembler::Condition)$cmp$$cmpcode);
8850 %}
8851
8852 ins_pipe(icond_reg_reg);
8853 %}
8854
8855 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8856 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8857
8858 ins_cost(INSN_COST * 2);
8859 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8860
8861 ins_encode %{
8862 __ cselw(as_Register($dst$$reg),
8863 as_Register($src2$$reg),
8864 as_Register($src1$$reg),
8865 (Assembler::Condition)$cmp$$cmpcode);
8866 %}
8867
8868 ins_pipe(icond_reg_reg);
8869 %}
8870
8871 // special cases where one arg is zero
8872
8873 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8874 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8875
8876 ins_cost(INSN_COST * 2);
8877 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8878
8879 ins_encode %{
8880 __ cselw(as_Register($dst$$reg),
8881 zr,
8882 as_Register($src$$reg),
8883 (Assembler::Condition)$cmp$$cmpcode);
8884 %}
8885
8886 ins_pipe(icond_reg);
8887 %}
8888
8889 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8890 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8891
8892 ins_cost(INSN_COST * 2);
8893 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8894
8895 ins_encode %{
8896 __ cselw(as_Register($dst$$reg),
8897 zr,
8898 as_Register($src$$reg),
8899 (Assembler::Condition)$cmp$$cmpcode);
8900 %}
8901
8902 ins_pipe(icond_reg);
8903 %}
8904
8905 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8906 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8907
8908 ins_cost(INSN_COST * 2);
8909 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8910
8911 ins_encode %{
8912 __ cselw(as_Register($dst$$reg),
8913 as_Register($src$$reg),
8914 zr,
8915 (Assembler::Condition)$cmp$$cmpcode);
8916 %}
8917
8918 ins_pipe(icond_reg);
8919 %}
8920
8921 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8922 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8923
8924 ins_cost(INSN_COST * 2);
8925 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8926
8927 ins_encode %{
8928 __ cselw(as_Register($dst$$reg),
8929 as_Register($src$$reg),
8930 zr,
8931 (Assembler::Condition)$cmp$$cmpcode);
8932 %}
8933
8934 ins_pipe(icond_reg);
8935 %}
8936
8937 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8938 %{
8939 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8940
8941 ins_cost(INSN_COST * 3);
8942
8943 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8944 ins_encode %{
8945 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8946 __ fcsels(as_FloatRegister($dst$$reg),
8947 as_FloatRegister($src2$$reg),
8948 as_FloatRegister($src1$$reg),
8949 cond);
8950 %}
8951
8952 ins_pipe(fp_cond_reg_reg_s);
8953 %}
8954
8955 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
8956 %{
8957 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8958
8959 ins_cost(INSN_COST * 3);
8960
8961 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8962 ins_encode %{
8963 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8964 __ fcsels(as_FloatRegister($dst$$reg),
8965 as_FloatRegister($src2$$reg),
8966 as_FloatRegister($src1$$reg),
8967 cond);
8968 %}
8969
8970 ins_pipe(fp_cond_reg_reg_s);
8971 %}
8972
8973 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
8974 %{
8975 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8976
8977 ins_cost(INSN_COST * 3);
8978
8979 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8980 ins_encode %{
8981 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8982 __ fcseld(as_FloatRegister($dst$$reg),
8983 as_FloatRegister($src2$$reg),
8984 as_FloatRegister($src1$$reg),
8985 cond);
8986 %}
8987
8988 ins_pipe(fp_cond_reg_reg_d);
8989 %}
8990
8991 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
8992 %{
8993 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8994
8995 ins_cost(INSN_COST * 3);
8996
8997 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8998 ins_encode %{
8999 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9000 __ fcseld(as_FloatRegister($dst$$reg),
9001 as_FloatRegister($src2$$reg),
9002 as_FloatRegister($src1$$reg),
9003 cond);
9004 %}
9005
9006 ins_pipe(fp_cond_reg_reg_d);
9007 %}
9008
9009 // ============================================================================
9010 // Arithmetic Instructions
9011 //
9012
9013 // Integer Addition
9014
9015 // TODO
9016 // these currently employ operations which do not set CR and hence are
9017 // not flagged as killing CR but we would like to isolate the cases
9018 // where we want to set flags from those where we don't. need to work
9019 // out how to do that.
9020
9021 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9022 match(Set dst (AddI src1 src2));
9023
9024 ins_cost(INSN_COST);
9025 format %{ "addw $dst, $src1, $src2" %}
9026
9027 ins_encode %{
9028 __ addw(as_Register($dst$$reg),
9029 as_Register($src1$$reg),
9030 as_Register($src2$$reg));
9031 %}
9032
9033 ins_pipe(ialu_reg_reg);
9034 %}
9035
9036 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9037 match(Set dst (AddI src1 src2));
9038
9039 ins_cost(INSN_COST);
9040 format %{ "addw $dst, $src1, $src2" %}
9041
9042 // use opcode to indicate that this is an add not a sub
9043 opcode(0x0);
9044
9045 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9046
9047 ins_pipe(ialu_reg_imm);
9048 %}
9049
9050 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9051 match(Set dst (AddI (ConvL2I src1) src2));
9052
9053 ins_cost(INSN_COST);
9054 format %{ "addw $dst, $src1, $src2" %}
9055
9056 // use opcode to indicate that this is an add not a sub
9057 opcode(0x0);
9058
9059 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9060
9061 ins_pipe(ialu_reg_imm);
9062 %}
9063
9064 // Pointer Addition
9065 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9066 match(Set dst (AddP src1 src2));
9067
9068 ins_cost(INSN_COST);
9069 format %{ "add $dst, $src1, $src2\t# ptr" %}
9070
9071 ins_encode %{
9072 __ add(as_Register($dst$$reg),
9073 as_Register($src1$$reg),
9074 as_Register($src2$$reg));
9075 %}
9076
9077 ins_pipe(ialu_reg_reg);
9078 %}
9079
9080 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9081 match(Set dst (AddP src1 (ConvI2L src2)));
9082
9083 ins_cost(1.9 * INSN_COST);
9084 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9085
9086 ins_encode %{
9087 __ add(as_Register($dst$$reg),
9088 as_Register($src1$$reg),
9089 as_Register($src2$$reg), ext::sxtw);
9090 %}
9091
9092 ins_pipe(ialu_reg_reg);
9093 %}
9094
9095 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9096 match(Set dst (AddP src1 (LShiftL src2 scale)));
9097
9098 ins_cost(1.9 * INSN_COST);
9099 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9100
9101 ins_encode %{
9102 __ lea(as_Register($dst$$reg),
9103 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9104 Address::lsl($scale$$constant)));
9105 %}
9106
9107 ins_pipe(ialu_reg_reg_shift);
9108 %}
9109
9110 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9111 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9112
9113 ins_cost(1.9 * INSN_COST);
9114 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9115
9116 ins_encode %{
9117 __ lea(as_Register($dst$$reg),
9118 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9119 Address::sxtw($scale$$constant)));
9120 %}
9121
9122 ins_pipe(ialu_reg_reg_shift);
9123 %}
9124
9125 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9126 match(Set dst (LShiftL (ConvI2L src) scale));
9127
9128 ins_cost(INSN_COST);
9129 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9130
9131 ins_encode %{
9132 __ sbfiz(as_Register($dst$$reg),
9133 as_Register($src$$reg),
9134 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9135 %}
9136
9137 ins_pipe(ialu_reg_shift);
9138 %}
9139
9140 // Pointer Immediate Addition
9141 // n.b. this needs to be more expensive than using an indirect memory
9142 // operand
9143 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9144 match(Set dst (AddP src1 src2));
9145
9146 ins_cost(INSN_COST);
9147 format %{ "add $dst, $src1, $src2\t# ptr" %}
9148
9149 // use opcode to indicate that this is an add not a sub
9150 opcode(0x0);
9151
9152 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9153
9154 ins_pipe(ialu_reg_imm);
9155 %}
9156
9157 // Long Addition
9158 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9159
9160 match(Set dst (AddL src1 src2));
9161
9162 ins_cost(INSN_COST);
9163 format %{ "add $dst, $src1, $src2" %}
9164
9165 ins_encode %{
9166 __ add(as_Register($dst$$reg),
9167 as_Register($src1$$reg),
9168 as_Register($src2$$reg));
9169 %}
9170
9171 ins_pipe(ialu_reg_reg);
9172 %}
9173
9174 // No constant pool entries requiredLong Immediate Addition.
9175 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9176 match(Set dst (AddL src1 src2));
9177
9178 ins_cost(INSN_COST);
9179 format %{ "add $dst, $src1, $src2" %}
9180
9181 // use opcode to indicate that this is an add not a sub
9182 opcode(0x0);
9183
9184 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9185
9186 ins_pipe(ialu_reg_imm);
9187 %}
9188
9189 // Integer Subtraction
9190 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9191 match(Set dst (SubI src1 src2));
9192
9193 ins_cost(INSN_COST);
9194 format %{ "subw $dst, $src1, $src2" %}
9195
9196 ins_encode %{
9197 __ subw(as_Register($dst$$reg),
9198 as_Register($src1$$reg),
9199 as_Register($src2$$reg));
9200 %}
9201
9202 ins_pipe(ialu_reg_reg);
9203 %}
9204
9205 // Immediate Subtraction
9206 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9207 match(Set dst (SubI src1 src2));
9208
9209 ins_cost(INSN_COST);
9210 format %{ "subw $dst, $src1, $src2" %}
9211
9212 // use opcode to indicate that this is a sub not an add
9213 opcode(0x1);
9214
9215 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9216
9217 ins_pipe(ialu_reg_imm);
9218 %}
9219
9220 // Long Subtraction
9221 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9222
9223 match(Set dst (SubL src1 src2));
9224
9225 ins_cost(INSN_COST);
9226 format %{ "sub $dst, $src1, $src2" %}
9227
9228 ins_encode %{
9229 __ sub(as_Register($dst$$reg),
9230 as_Register($src1$$reg),
9231 as_Register($src2$$reg));
9232 %}
9233
9234 ins_pipe(ialu_reg_reg);
9235 %}
9236
9237 // No constant pool entries requiredLong Immediate Subtraction.
9238 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9239 match(Set dst (SubL src1 src2));
9240
9241 ins_cost(INSN_COST);
9242 format %{ "sub$dst, $src1, $src2" %}
9243
9244 // use opcode to indicate that this is a sub not an add
9245 opcode(0x1);
9246
9247 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9248
9249 ins_pipe(ialu_reg_imm);
9250 %}
9251
9252 // Integer Negation (special case for sub)
9253
9254 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9255 match(Set dst (SubI zero src));
9256
9257 ins_cost(INSN_COST);
9258 format %{ "negw $dst, $src\t# int" %}
9259
9260 ins_encode %{
9261 __ negw(as_Register($dst$$reg),
9262 as_Register($src$$reg));
9263 %}
9264
9265 ins_pipe(ialu_reg);
9266 %}
9267
9268 // Long Negation
9269
9270 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9271 match(Set dst (SubL zero src));
9272
9273 ins_cost(INSN_COST);
9274 format %{ "neg $dst, $src\t# long" %}
9275
9276 ins_encode %{
9277 __ neg(as_Register($dst$$reg),
9278 as_Register($src$$reg));
9279 %}
9280
9281 ins_pipe(ialu_reg);
9282 %}
9283
9284 // Integer Multiply
9285
9286 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9287 match(Set dst (MulI src1 src2));
9288
9289 ins_cost(INSN_COST * 3);
9290 format %{ "mulw $dst, $src1, $src2" %}
9291
9292 ins_encode %{
9293 __ mulw(as_Register($dst$$reg),
9294 as_Register($src1$$reg),
9295 as_Register($src2$$reg));
9296 %}
9297
9298 ins_pipe(imul_reg_reg);
9299 %}
9300
9301 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9302 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9303
9304 ins_cost(INSN_COST * 3);
9305 format %{ "smull $dst, $src1, $src2" %}
9306
9307 ins_encode %{
9308 __ smull(as_Register($dst$$reg),
9309 as_Register($src1$$reg),
9310 as_Register($src2$$reg));
9311 %}
9312
9313 ins_pipe(imul_reg_reg);
9314 %}
9315
9316 // Long Multiply
9317
9318 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9319 match(Set dst (MulL src1 src2));
9320
9321 ins_cost(INSN_COST * 5);
9322 format %{ "mul $dst, $src1, $src2" %}
9323
9324 ins_encode %{
9325 __ mul(as_Register($dst$$reg),
9326 as_Register($src1$$reg),
9327 as_Register($src2$$reg));
9328 %}
9329
9330 ins_pipe(lmul_reg_reg);
9331 %}
9332
9333 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9334 %{
9335 match(Set dst (MulHiL src1 src2));
9336
9337 ins_cost(INSN_COST * 7);
9338 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9339
9340 ins_encode %{
9341 __ smulh(as_Register($dst$$reg),
9342 as_Register($src1$$reg),
9343 as_Register($src2$$reg));
9344 %}
9345
9346 ins_pipe(lmul_reg_reg);
9347 %}
9348
9349 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9350 %{
9351 match(Set dst (UMulHiL src1 src2));
9352
9353 ins_cost(INSN_COST * 7);
9354 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9355
9356 ins_encode %{
9357 __ umulh(as_Register($dst$$reg),
9358 as_Register($src1$$reg),
9359 as_Register($src2$$reg));
9360 %}
9361
9362 ins_pipe(lmul_reg_reg);
9363 %}
9364
9365 // Combined Integer Multiply & Add/Sub
9366
9367 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9368 match(Set dst (AddI src3 (MulI src1 src2)));
9369
9370 ins_cost(INSN_COST * 3);
9371 format %{ "madd $dst, $src1, $src2, $src3" %}
9372
9373 ins_encode %{
9374 __ maddw(as_Register($dst$$reg),
9375 as_Register($src1$$reg),
9376 as_Register($src2$$reg),
9377 as_Register($src3$$reg));
9378 %}
9379
9380 ins_pipe(imac_reg_reg);
9381 %}
9382
9383 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9384 match(Set dst (SubI src3 (MulI src1 src2)));
9385
9386 ins_cost(INSN_COST * 3);
9387 format %{ "msub $dst, $src1, $src2, $src3" %}
9388
9389 ins_encode %{
9390 __ msubw(as_Register($dst$$reg),
9391 as_Register($src1$$reg),
9392 as_Register($src2$$reg),
9393 as_Register($src3$$reg));
9394 %}
9395
9396 ins_pipe(imac_reg_reg);
9397 %}
9398
9399 // Combined Integer Multiply & Neg
9400
9401 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9402 match(Set dst (MulI (SubI zero src1) src2));
9403
9404 ins_cost(INSN_COST * 3);
9405 format %{ "mneg $dst, $src1, $src2" %}
9406
9407 ins_encode %{
9408 __ mnegw(as_Register($dst$$reg),
9409 as_Register($src1$$reg),
9410 as_Register($src2$$reg));
9411 %}
9412
9413 ins_pipe(imac_reg_reg);
9414 %}
9415
9416 // Combined Long Multiply & Add/Sub
9417
9418 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9419 match(Set dst (AddL src3 (MulL src1 src2)));
9420
9421 ins_cost(INSN_COST * 5);
9422 format %{ "madd $dst, $src1, $src2, $src3" %}
9423
9424 ins_encode %{
9425 __ madd(as_Register($dst$$reg),
9426 as_Register($src1$$reg),
9427 as_Register($src2$$reg),
9428 as_Register($src3$$reg));
9429 %}
9430
9431 ins_pipe(lmac_reg_reg);
9432 %}
9433
9434 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9435 match(Set dst (SubL src3 (MulL src1 src2)));
9436
9437 ins_cost(INSN_COST * 5);
9438 format %{ "msub $dst, $src1, $src2, $src3" %}
9439
9440 ins_encode %{
9441 __ msub(as_Register($dst$$reg),
9442 as_Register($src1$$reg),
9443 as_Register($src2$$reg),
9444 as_Register($src3$$reg));
9445 %}
9446
9447 ins_pipe(lmac_reg_reg);
9448 %}
9449
9450 // Combined Long Multiply & Neg
9451
9452 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9453 match(Set dst (MulL (SubL zero src1) src2));
9454
9455 ins_cost(INSN_COST * 5);
9456 format %{ "mneg $dst, $src1, $src2" %}
9457
9458 ins_encode %{
9459 __ mneg(as_Register($dst$$reg),
9460 as_Register($src1$$reg),
9461 as_Register($src2$$reg));
9462 %}
9463
9464 ins_pipe(lmac_reg_reg);
9465 %}
9466
9467 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9468
9469 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9470 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9471
9472 ins_cost(INSN_COST * 3);
9473 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9474
9475 ins_encode %{
9476 __ smaddl(as_Register($dst$$reg),
9477 as_Register($src1$$reg),
9478 as_Register($src2$$reg),
9479 as_Register($src3$$reg));
9480 %}
9481
9482 ins_pipe(imac_reg_reg);
9483 %}
9484
9485 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9486 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9487
9488 ins_cost(INSN_COST * 3);
9489 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9490
9491 ins_encode %{
9492 __ smsubl(as_Register($dst$$reg),
9493 as_Register($src1$$reg),
9494 as_Register($src2$$reg),
9495 as_Register($src3$$reg));
9496 %}
9497
9498 ins_pipe(imac_reg_reg);
9499 %}
9500
9501 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9502 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9503
9504 ins_cost(INSN_COST * 3);
9505 format %{ "smnegl $dst, $src1, $src2" %}
9506
9507 ins_encode %{
9508 __ smnegl(as_Register($dst$$reg),
9509 as_Register($src1$$reg),
9510 as_Register($src2$$reg));
9511 %}
9512
9513 ins_pipe(imac_reg_reg);
9514 %}
9515
9516 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9517
9518 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9519 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9520
9521 ins_cost(INSN_COST * 5);
9522 format %{ "mulw rscratch1, $src1, $src2\n\t"
9523 "maddw $dst, $src3, $src4, rscratch1" %}
9524
9525 ins_encode %{
9526 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9527 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9528
9529 ins_pipe(imac_reg_reg);
9530 %}
9531
9532 // Integer Divide
9533
9534 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9535 match(Set dst (DivI src1 src2));
9536
9537 ins_cost(INSN_COST * 19);
9538 format %{ "sdivw $dst, $src1, $src2" %}
9539
9540 ins_encode(aarch64_enc_divw(dst, src1, src2));
9541 ins_pipe(idiv_reg_reg);
9542 %}
9543
9544 // Long Divide
9545
9546 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9547 match(Set dst (DivL src1 src2));
9548
9549 ins_cost(INSN_COST * 35);
9550 format %{ "sdiv $dst, $src1, $src2" %}
9551
9552 ins_encode(aarch64_enc_div(dst, src1, src2));
9553 ins_pipe(ldiv_reg_reg);
9554 %}
9555
9556 // Integer Remainder
9557
9558 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9559 match(Set dst (ModI src1 src2));
9560
9561 ins_cost(INSN_COST * 22);
9562 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9563 "msubw $dst, rscratch1, $src2, $src1" %}
9564
9565 ins_encode(aarch64_enc_modw(dst, src1, src2));
9566 ins_pipe(idiv_reg_reg);
9567 %}
9568
9569 // Long Remainder
9570
9571 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9572 match(Set dst (ModL src1 src2));
9573
9574 ins_cost(INSN_COST * 38);
9575 format %{ "sdiv rscratch1, $src1, $src2\n"
9576 "msub $dst, rscratch1, $src2, $src1" %}
9577
9578 ins_encode(aarch64_enc_mod(dst, src1, src2));
9579 ins_pipe(ldiv_reg_reg);
9580 %}
9581
9582 // Unsigned Integer Divide
9583
9584 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9585 match(Set dst (UDivI src1 src2));
9586
9587 ins_cost(INSN_COST * 19);
9588 format %{ "udivw $dst, $src1, $src2" %}
9589
9590 ins_encode %{
9591 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9592 %}
9593
9594 ins_pipe(idiv_reg_reg);
9595 %}
9596
9597 // Unsigned Long Divide
9598
9599 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9600 match(Set dst (UDivL src1 src2));
9601
9602 ins_cost(INSN_COST * 35);
9603 format %{ "udiv $dst, $src1, $src2" %}
9604
9605 ins_encode %{
9606 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9607 %}
9608
9609 ins_pipe(ldiv_reg_reg);
9610 %}
9611
9612 // Unsigned Integer Remainder
9613
9614 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9615 match(Set dst (UModI src1 src2));
9616
9617 ins_cost(INSN_COST * 22);
9618 format %{ "udivw rscratch1, $src1, $src2\n\t"
9619 "msubw $dst, rscratch1, $src2, $src1" %}
9620
9621 ins_encode %{
9622 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9623 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9624 %}
9625
9626 ins_pipe(idiv_reg_reg);
9627 %}
9628
9629 // Unsigned Long Remainder
9630
9631 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9632 match(Set dst (UModL src1 src2));
9633
9634 ins_cost(INSN_COST * 38);
9635 format %{ "udiv rscratch1, $src1, $src2\n"
9636 "msub $dst, rscratch1, $src2, $src1" %}
9637
9638 ins_encode %{
9639 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9640 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9641 %}
9642
9643 ins_pipe(ldiv_reg_reg);
9644 %}
9645
9646 // Integer Shifts
9647
9648 // Shift Left Register
9649 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9650 match(Set dst (LShiftI src1 src2));
9651
9652 ins_cost(INSN_COST * 2);
9653 format %{ "lslvw $dst, $src1, $src2" %}
9654
9655 ins_encode %{
9656 __ lslvw(as_Register($dst$$reg),
9657 as_Register($src1$$reg),
9658 as_Register($src2$$reg));
9659 %}
9660
9661 ins_pipe(ialu_reg_reg_vshift);
9662 %}
9663
9664 // Shift Left Immediate
9665 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9666 match(Set dst (LShiftI src1 src2));
9667
9668 ins_cost(INSN_COST);
9669 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9670
9671 ins_encode %{
9672 __ lslw(as_Register($dst$$reg),
9673 as_Register($src1$$reg),
9674 $src2$$constant & 0x1f);
9675 %}
9676
9677 ins_pipe(ialu_reg_shift);
9678 %}
9679
9680 // Shift Right Logical Register
9681 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9682 match(Set dst (URShiftI src1 src2));
9683
9684 ins_cost(INSN_COST * 2);
9685 format %{ "lsrvw $dst, $src1, $src2" %}
9686
9687 ins_encode %{
9688 __ lsrvw(as_Register($dst$$reg),
9689 as_Register($src1$$reg),
9690 as_Register($src2$$reg));
9691 %}
9692
9693 ins_pipe(ialu_reg_reg_vshift);
9694 %}
9695
9696 // Shift Right Logical Immediate
9697 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9698 match(Set dst (URShiftI src1 src2));
9699
9700 ins_cost(INSN_COST);
9701 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9702
9703 ins_encode %{
9704 __ lsrw(as_Register($dst$$reg),
9705 as_Register($src1$$reg),
9706 $src2$$constant & 0x1f);
9707 %}
9708
9709 ins_pipe(ialu_reg_shift);
9710 %}
9711
9712 // Shift Right Arithmetic Register
9713 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9714 match(Set dst (RShiftI src1 src2));
9715
9716 ins_cost(INSN_COST * 2);
9717 format %{ "asrvw $dst, $src1, $src2" %}
9718
9719 ins_encode %{
9720 __ asrvw(as_Register($dst$$reg),
9721 as_Register($src1$$reg),
9722 as_Register($src2$$reg));
9723 %}
9724
9725 ins_pipe(ialu_reg_reg_vshift);
9726 %}
9727
9728 // Shift Right Arithmetic Immediate
9729 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9730 match(Set dst (RShiftI src1 src2));
9731
9732 ins_cost(INSN_COST);
9733 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9734
9735 ins_encode %{
9736 __ asrw(as_Register($dst$$reg),
9737 as_Register($src1$$reg),
9738 $src2$$constant & 0x1f);
9739 %}
9740
9741 ins_pipe(ialu_reg_shift);
9742 %}
9743
9744 // Combined Int Mask and Right Shift (using UBFM)
9745 // TODO
9746
9747 // Long Shifts
9748
9749 // Shift Left Register
9750 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9751 match(Set dst (LShiftL src1 src2));
9752
9753 ins_cost(INSN_COST * 2);
9754 format %{ "lslv $dst, $src1, $src2" %}
9755
9756 ins_encode %{
9757 __ lslv(as_Register($dst$$reg),
9758 as_Register($src1$$reg),
9759 as_Register($src2$$reg));
9760 %}
9761
9762 ins_pipe(ialu_reg_reg_vshift);
9763 %}
9764
9765 // Shift Left Immediate
9766 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9767 match(Set dst (LShiftL src1 src2));
9768
9769 ins_cost(INSN_COST);
9770 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9771
9772 ins_encode %{
9773 __ lsl(as_Register($dst$$reg),
9774 as_Register($src1$$reg),
9775 $src2$$constant & 0x3f);
9776 %}
9777
9778 ins_pipe(ialu_reg_shift);
9779 %}
9780
9781 // Shift Right Logical Register
9782 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9783 match(Set dst (URShiftL src1 src2));
9784
9785 ins_cost(INSN_COST * 2);
9786 format %{ "lsrv $dst, $src1, $src2" %}
9787
9788 ins_encode %{
9789 __ lsrv(as_Register($dst$$reg),
9790 as_Register($src1$$reg),
9791 as_Register($src2$$reg));
9792 %}
9793
9794 ins_pipe(ialu_reg_reg_vshift);
9795 %}
9796
9797 // Shift Right Logical Immediate
9798 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9799 match(Set dst (URShiftL src1 src2));
9800
9801 ins_cost(INSN_COST);
9802 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9803
9804 ins_encode %{
9805 __ lsr(as_Register($dst$$reg),
9806 as_Register($src1$$reg),
9807 $src2$$constant & 0x3f);
9808 %}
9809
9810 ins_pipe(ialu_reg_shift);
9811 %}
9812
9813 // A special-case pattern for card table stores.
9814 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9815 match(Set dst (URShiftL (CastP2X src1) src2));
9816
9817 ins_cost(INSN_COST);
9818 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9819
9820 ins_encode %{
9821 __ lsr(as_Register($dst$$reg),
9822 as_Register($src1$$reg),
9823 $src2$$constant & 0x3f);
9824 %}
9825
9826 ins_pipe(ialu_reg_shift);
9827 %}
9828
9829 // Shift Right Arithmetic Register
9830 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9831 match(Set dst (RShiftL src1 src2));
9832
9833 ins_cost(INSN_COST * 2);
9834 format %{ "asrv $dst, $src1, $src2" %}
9835
9836 ins_encode %{
9837 __ asrv(as_Register($dst$$reg),
9838 as_Register($src1$$reg),
9839 as_Register($src2$$reg));
9840 %}
9841
9842 ins_pipe(ialu_reg_reg_vshift);
9843 %}
9844
9845 // Shift Right Arithmetic Immediate
9846 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9847 match(Set dst (RShiftL src1 src2));
9848
9849 ins_cost(INSN_COST);
9850 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9851
9852 ins_encode %{
9853 __ asr(as_Register($dst$$reg),
9854 as_Register($src1$$reg),
9855 $src2$$constant & 0x3f);
9856 %}
9857
9858 ins_pipe(ialu_reg_shift);
9859 %}
9860
9861 // BEGIN This section of the file is automatically generated. Do not edit --------------
9862 // This section is generated from aarch64_ad.m4
9863
9864 // This pattern is automatically generated from aarch64_ad.m4
9865 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9866 instruct regL_not_reg(iRegLNoSp dst,
9867 iRegL src1, immL_M1 m1,
9868 rFlagsReg cr) %{
9869 match(Set dst (XorL src1 m1));
9870 ins_cost(INSN_COST);
9871 format %{ "eon $dst, $src1, zr" %}
9872
9873 ins_encode %{
9874 __ eon(as_Register($dst$$reg),
9875 as_Register($src1$$reg),
9876 zr,
9877 Assembler::LSL, 0);
9878 %}
9879
9880 ins_pipe(ialu_reg);
9881 %}
9882
9883 // This pattern is automatically generated from aarch64_ad.m4
9884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9885 instruct regI_not_reg(iRegINoSp dst,
9886 iRegIorL2I src1, immI_M1 m1,
9887 rFlagsReg cr) %{
9888 match(Set dst (XorI src1 m1));
9889 ins_cost(INSN_COST);
9890 format %{ "eonw $dst, $src1, zr" %}
9891
9892 ins_encode %{
9893 __ eonw(as_Register($dst$$reg),
9894 as_Register($src1$$reg),
9895 zr,
9896 Assembler::LSL, 0);
9897 %}
9898
9899 ins_pipe(ialu_reg);
9900 %}
9901
9902 // This pattern is automatically generated from aarch64_ad.m4
9903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9904 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9905 immI0 zero, iRegIorL2I src1, immI src2) %{
9906 match(Set dst (SubI zero (URShiftI src1 src2)));
9907
9908 ins_cost(1.9 * INSN_COST);
9909 format %{ "negw $dst, $src1, LSR $src2" %}
9910
9911 ins_encode %{
9912 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9913 Assembler::LSR, $src2$$constant & 0x1f);
9914 %}
9915
9916 ins_pipe(ialu_reg_shift);
9917 %}
9918
9919 // This pattern is automatically generated from aarch64_ad.m4
9920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9921 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9922 immI0 zero, iRegIorL2I src1, immI src2) %{
9923 match(Set dst (SubI zero (RShiftI src1 src2)));
9924
9925 ins_cost(1.9 * INSN_COST);
9926 format %{ "negw $dst, $src1, ASR $src2" %}
9927
9928 ins_encode %{
9929 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9930 Assembler::ASR, $src2$$constant & 0x1f);
9931 %}
9932
9933 ins_pipe(ialu_reg_shift);
9934 %}
9935
9936 // This pattern is automatically generated from aarch64_ad.m4
9937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9938 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9939 immI0 zero, iRegIorL2I src1, immI src2) %{
9940 match(Set dst (SubI zero (LShiftI src1 src2)));
9941
9942 ins_cost(1.9 * INSN_COST);
9943 format %{ "negw $dst, $src1, LSL $src2" %}
9944
9945 ins_encode %{
9946 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9947 Assembler::LSL, $src2$$constant & 0x1f);
9948 %}
9949
9950 ins_pipe(ialu_reg_shift);
9951 %}
9952
9953 // This pattern is automatically generated from aarch64_ad.m4
9954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9955 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
9956 immL0 zero, iRegL src1, immI src2) %{
9957 match(Set dst (SubL zero (URShiftL src1 src2)));
9958
9959 ins_cost(1.9 * INSN_COST);
9960 format %{ "neg $dst, $src1, LSR $src2" %}
9961
9962 ins_encode %{
9963 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9964 Assembler::LSR, $src2$$constant & 0x3f);
9965 %}
9966
9967 ins_pipe(ialu_reg_shift);
9968 %}
9969
9970 // This pattern is automatically generated from aarch64_ad.m4
9971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9972 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
9973 immL0 zero, iRegL src1, immI src2) %{
9974 match(Set dst (SubL zero (RShiftL src1 src2)));
9975
9976 ins_cost(1.9 * INSN_COST);
9977 format %{ "neg $dst, $src1, ASR $src2" %}
9978
9979 ins_encode %{
9980 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9981 Assembler::ASR, $src2$$constant & 0x3f);
9982 %}
9983
9984 ins_pipe(ialu_reg_shift);
9985 %}
9986
9987 // This pattern is automatically generated from aarch64_ad.m4
9988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9989 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
9990 immL0 zero, iRegL src1, immI src2) %{
9991 match(Set dst (SubL zero (LShiftL src1 src2)));
9992
9993 ins_cost(1.9 * INSN_COST);
9994 format %{ "neg $dst, $src1, LSL $src2" %}
9995
9996 ins_encode %{
9997 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9998 Assembler::LSL, $src2$$constant & 0x3f);
9999 %}
10000
10001 ins_pipe(ialu_reg_shift);
10002 %}
10003
10004 // This pattern is automatically generated from aarch64_ad.m4
10005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10006 instruct AndI_reg_not_reg(iRegINoSp dst,
10007 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10008 match(Set dst (AndI src1 (XorI src2 m1)));
10009 ins_cost(INSN_COST);
10010 format %{ "bicw $dst, $src1, $src2" %}
10011
10012 ins_encode %{
10013 __ bicw(as_Register($dst$$reg),
10014 as_Register($src1$$reg),
10015 as_Register($src2$$reg),
10016 Assembler::LSL, 0);
10017 %}
10018
10019 ins_pipe(ialu_reg_reg);
10020 %}
10021
10022 // This pattern is automatically generated from aarch64_ad.m4
10023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10024 instruct AndL_reg_not_reg(iRegLNoSp dst,
10025 iRegL src1, iRegL src2, immL_M1 m1) %{
10026 match(Set dst (AndL src1 (XorL src2 m1)));
10027 ins_cost(INSN_COST);
10028 format %{ "bic $dst, $src1, $src2" %}
10029
10030 ins_encode %{
10031 __ bic(as_Register($dst$$reg),
10032 as_Register($src1$$reg),
10033 as_Register($src2$$reg),
10034 Assembler::LSL, 0);
10035 %}
10036
10037 ins_pipe(ialu_reg_reg);
10038 %}
10039
10040 // This pattern is automatically generated from aarch64_ad.m4
10041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10042 instruct OrI_reg_not_reg(iRegINoSp dst,
10043 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10044 match(Set dst (OrI src1 (XorI src2 m1)));
10045 ins_cost(INSN_COST);
10046 format %{ "ornw $dst, $src1, $src2" %}
10047
10048 ins_encode %{
10049 __ ornw(as_Register($dst$$reg),
10050 as_Register($src1$$reg),
10051 as_Register($src2$$reg),
10052 Assembler::LSL, 0);
10053 %}
10054
10055 ins_pipe(ialu_reg_reg);
10056 %}
10057
10058 // This pattern is automatically generated from aarch64_ad.m4
10059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10060 instruct OrL_reg_not_reg(iRegLNoSp dst,
10061 iRegL src1, iRegL src2, immL_M1 m1) %{
10062 match(Set dst (OrL src1 (XorL src2 m1)));
10063 ins_cost(INSN_COST);
10064 format %{ "orn $dst, $src1, $src2" %}
10065
10066 ins_encode %{
10067 __ orn(as_Register($dst$$reg),
10068 as_Register($src1$$reg),
10069 as_Register($src2$$reg),
10070 Assembler::LSL, 0);
10071 %}
10072
10073 ins_pipe(ialu_reg_reg);
10074 %}
10075
10076 // This pattern is automatically generated from aarch64_ad.m4
10077 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10078 instruct XorI_reg_not_reg(iRegINoSp dst,
10079 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10080 match(Set dst (XorI m1 (XorI src2 src1)));
10081 ins_cost(INSN_COST);
10082 format %{ "eonw $dst, $src1, $src2" %}
10083
10084 ins_encode %{
10085 __ eonw(as_Register($dst$$reg),
10086 as_Register($src1$$reg),
10087 as_Register($src2$$reg),
10088 Assembler::LSL, 0);
10089 %}
10090
10091 ins_pipe(ialu_reg_reg);
10092 %}
10093
10094 // This pattern is automatically generated from aarch64_ad.m4
10095 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10096 instruct XorL_reg_not_reg(iRegLNoSp dst,
10097 iRegL src1, iRegL src2, immL_M1 m1) %{
10098 match(Set dst (XorL m1 (XorL src2 src1)));
10099 ins_cost(INSN_COST);
10100 format %{ "eon $dst, $src1, $src2" %}
10101
10102 ins_encode %{
10103 __ eon(as_Register($dst$$reg),
10104 as_Register($src1$$reg),
10105 as_Register($src2$$reg),
10106 Assembler::LSL, 0);
10107 %}
10108
10109 ins_pipe(ialu_reg_reg);
10110 %}
10111
10112 // This pattern is automatically generated from aarch64_ad.m4
10113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10114 // val & (-1 ^ (val >>> shift)) ==> bicw
10115 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10116 iRegIorL2I src1, iRegIorL2I src2,
10117 immI src3, immI_M1 src4) %{
10118 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10119 ins_cost(1.9 * INSN_COST);
10120 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10121
10122 ins_encode %{
10123 __ bicw(as_Register($dst$$reg),
10124 as_Register($src1$$reg),
10125 as_Register($src2$$reg),
10126 Assembler::LSR,
10127 $src3$$constant & 0x1f);
10128 %}
10129
10130 ins_pipe(ialu_reg_reg_shift);
10131 %}
10132
10133 // This pattern is automatically generated from aarch64_ad.m4
10134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10135 // val & (-1 ^ (val >>> shift)) ==> bic
10136 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10137 iRegL src1, iRegL src2,
10138 immI src3, immL_M1 src4) %{
10139 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10140 ins_cost(1.9 * INSN_COST);
10141 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10142
10143 ins_encode %{
10144 __ bic(as_Register($dst$$reg),
10145 as_Register($src1$$reg),
10146 as_Register($src2$$reg),
10147 Assembler::LSR,
10148 $src3$$constant & 0x3f);
10149 %}
10150
10151 ins_pipe(ialu_reg_reg_shift);
10152 %}
10153
10154 // This pattern is automatically generated from aarch64_ad.m4
10155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10156 // val & (-1 ^ (val >> shift)) ==> bicw
10157 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10158 iRegIorL2I src1, iRegIorL2I src2,
10159 immI src3, immI_M1 src4) %{
10160 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10161 ins_cost(1.9 * INSN_COST);
10162 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10163
10164 ins_encode %{
10165 __ bicw(as_Register($dst$$reg),
10166 as_Register($src1$$reg),
10167 as_Register($src2$$reg),
10168 Assembler::ASR,
10169 $src3$$constant & 0x1f);
10170 %}
10171
10172 ins_pipe(ialu_reg_reg_shift);
10173 %}
10174
10175 // This pattern is automatically generated from aarch64_ad.m4
10176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10177 // val & (-1 ^ (val >> shift)) ==> bic
10178 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10179 iRegL src1, iRegL src2,
10180 immI src3, immL_M1 src4) %{
10181 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10182 ins_cost(1.9 * INSN_COST);
10183 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10184
10185 ins_encode %{
10186 __ bic(as_Register($dst$$reg),
10187 as_Register($src1$$reg),
10188 as_Register($src2$$reg),
10189 Assembler::ASR,
10190 $src3$$constant & 0x3f);
10191 %}
10192
10193 ins_pipe(ialu_reg_reg_shift);
10194 %}
10195
10196 // This pattern is automatically generated from aarch64_ad.m4
10197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10198 // val & (-1 ^ (val ror shift)) ==> bicw
10199 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10200 iRegIorL2I src1, iRegIorL2I src2,
10201 immI src3, immI_M1 src4) %{
10202 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10203 ins_cost(1.9 * INSN_COST);
10204 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10205
10206 ins_encode %{
10207 __ bicw(as_Register($dst$$reg),
10208 as_Register($src1$$reg),
10209 as_Register($src2$$reg),
10210 Assembler::ROR,
10211 $src3$$constant & 0x1f);
10212 %}
10213
10214 ins_pipe(ialu_reg_reg_shift);
10215 %}
10216
10217 // This pattern is automatically generated from aarch64_ad.m4
10218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10219 // val & (-1 ^ (val ror shift)) ==> bic
10220 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10221 iRegL src1, iRegL src2,
10222 immI src3, immL_M1 src4) %{
10223 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10224 ins_cost(1.9 * INSN_COST);
10225 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10226
10227 ins_encode %{
10228 __ bic(as_Register($dst$$reg),
10229 as_Register($src1$$reg),
10230 as_Register($src2$$reg),
10231 Assembler::ROR,
10232 $src3$$constant & 0x3f);
10233 %}
10234
10235 ins_pipe(ialu_reg_reg_shift);
10236 %}
10237
10238 // This pattern is automatically generated from aarch64_ad.m4
10239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10240 // val & (-1 ^ (val << shift)) ==> bicw
10241 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10242 iRegIorL2I src1, iRegIorL2I src2,
10243 immI src3, immI_M1 src4) %{
10244 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10245 ins_cost(1.9 * INSN_COST);
10246 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10247
10248 ins_encode %{
10249 __ bicw(as_Register($dst$$reg),
10250 as_Register($src1$$reg),
10251 as_Register($src2$$reg),
10252 Assembler::LSL,
10253 $src3$$constant & 0x1f);
10254 %}
10255
10256 ins_pipe(ialu_reg_reg_shift);
10257 %}
10258
10259 // This pattern is automatically generated from aarch64_ad.m4
10260 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10261 // val & (-1 ^ (val << shift)) ==> bic
10262 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10263 iRegL src1, iRegL src2,
10264 immI src3, immL_M1 src4) %{
10265 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10266 ins_cost(1.9 * INSN_COST);
10267 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10268
10269 ins_encode %{
10270 __ bic(as_Register($dst$$reg),
10271 as_Register($src1$$reg),
10272 as_Register($src2$$reg),
10273 Assembler::LSL,
10274 $src3$$constant & 0x3f);
10275 %}
10276
10277 ins_pipe(ialu_reg_reg_shift);
10278 %}
10279
10280 // This pattern is automatically generated from aarch64_ad.m4
10281 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10282 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10283 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10284 iRegIorL2I src1, iRegIorL2I src2,
10285 immI src3, immI_M1 src4) %{
10286 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10287 ins_cost(1.9 * INSN_COST);
10288 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10289
10290 ins_encode %{
10291 __ eonw(as_Register($dst$$reg),
10292 as_Register($src1$$reg),
10293 as_Register($src2$$reg),
10294 Assembler::LSR,
10295 $src3$$constant & 0x1f);
10296 %}
10297
10298 ins_pipe(ialu_reg_reg_shift);
10299 %}
10300
10301 // This pattern is automatically generated from aarch64_ad.m4
10302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10303 // val ^ (-1 ^ (val >>> shift)) ==> eon
10304 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10305 iRegL src1, iRegL src2,
10306 immI src3, immL_M1 src4) %{
10307 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10308 ins_cost(1.9 * INSN_COST);
10309 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10310
10311 ins_encode %{
10312 __ eon(as_Register($dst$$reg),
10313 as_Register($src1$$reg),
10314 as_Register($src2$$reg),
10315 Assembler::LSR,
10316 $src3$$constant & 0x3f);
10317 %}
10318
10319 ins_pipe(ialu_reg_reg_shift);
10320 %}
10321
10322 // This pattern is automatically generated from aarch64_ad.m4
10323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10324 // val ^ (-1 ^ (val >> shift)) ==> eonw
10325 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10326 iRegIorL2I src1, iRegIorL2I src2,
10327 immI src3, immI_M1 src4) %{
10328 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10329 ins_cost(1.9 * INSN_COST);
10330 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10331
10332 ins_encode %{
10333 __ eonw(as_Register($dst$$reg),
10334 as_Register($src1$$reg),
10335 as_Register($src2$$reg),
10336 Assembler::ASR,
10337 $src3$$constant & 0x1f);
10338 %}
10339
10340 ins_pipe(ialu_reg_reg_shift);
10341 %}
10342
10343 // This pattern is automatically generated from aarch64_ad.m4
10344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10345 // val ^ (-1 ^ (val >> shift)) ==> eon
10346 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10347 iRegL src1, iRegL src2,
10348 immI src3, immL_M1 src4) %{
10349 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10350 ins_cost(1.9 * INSN_COST);
10351 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10352
10353 ins_encode %{
10354 __ eon(as_Register($dst$$reg),
10355 as_Register($src1$$reg),
10356 as_Register($src2$$reg),
10357 Assembler::ASR,
10358 $src3$$constant & 0x3f);
10359 %}
10360
10361 ins_pipe(ialu_reg_reg_shift);
10362 %}
10363
10364 // This pattern is automatically generated from aarch64_ad.m4
10365 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10366 // val ^ (-1 ^ (val ror shift)) ==> eonw
10367 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10368 iRegIorL2I src1, iRegIorL2I src2,
10369 immI src3, immI_M1 src4) %{
10370 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10371 ins_cost(1.9 * INSN_COST);
10372 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10373
10374 ins_encode %{
10375 __ eonw(as_Register($dst$$reg),
10376 as_Register($src1$$reg),
10377 as_Register($src2$$reg),
10378 Assembler::ROR,
10379 $src3$$constant & 0x1f);
10380 %}
10381
10382 ins_pipe(ialu_reg_reg_shift);
10383 %}
10384
10385 // This pattern is automatically generated from aarch64_ad.m4
10386 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10387 // val ^ (-1 ^ (val ror shift)) ==> eon
10388 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10389 iRegL src1, iRegL src2,
10390 immI src3, immL_M1 src4) %{
10391 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10392 ins_cost(1.9 * INSN_COST);
10393 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10394
10395 ins_encode %{
10396 __ eon(as_Register($dst$$reg),
10397 as_Register($src1$$reg),
10398 as_Register($src2$$reg),
10399 Assembler::ROR,
10400 $src3$$constant & 0x3f);
10401 %}
10402
10403 ins_pipe(ialu_reg_reg_shift);
10404 %}
10405
10406 // This pattern is automatically generated from aarch64_ad.m4
10407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10408 // val ^ (-1 ^ (val << shift)) ==> eonw
10409 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10410 iRegIorL2I src1, iRegIorL2I src2,
10411 immI src3, immI_M1 src4) %{
10412 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10413 ins_cost(1.9 * INSN_COST);
10414 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10415
10416 ins_encode %{
10417 __ eonw(as_Register($dst$$reg),
10418 as_Register($src1$$reg),
10419 as_Register($src2$$reg),
10420 Assembler::LSL,
10421 $src3$$constant & 0x1f);
10422 %}
10423
10424 ins_pipe(ialu_reg_reg_shift);
10425 %}
10426
10427 // This pattern is automatically generated from aarch64_ad.m4
10428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10429 // val ^ (-1 ^ (val << shift)) ==> eon
10430 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10431 iRegL src1, iRegL src2,
10432 immI src3, immL_M1 src4) %{
10433 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10434 ins_cost(1.9 * INSN_COST);
10435 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10436
10437 ins_encode %{
10438 __ eon(as_Register($dst$$reg),
10439 as_Register($src1$$reg),
10440 as_Register($src2$$reg),
10441 Assembler::LSL,
10442 $src3$$constant & 0x3f);
10443 %}
10444
10445 ins_pipe(ialu_reg_reg_shift);
10446 %}
10447
10448 // This pattern is automatically generated from aarch64_ad.m4
10449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10450 // val | (-1 ^ (val >>> shift)) ==> ornw
10451 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10452 iRegIorL2I src1, iRegIorL2I src2,
10453 immI src3, immI_M1 src4) %{
10454 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10455 ins_cost(1.9 * INSN_COST);
10456 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10457
10458 ins_encode %{
10459 __ ornw(as_Register($dst$$reg),
10460 as_Register($src1$$reg),
10461 as_Register($src2$$reg),
10462 Assembler::LSR,
10463 $src3$$constant & 0x1f);
10464 %}
10465
10466 ins_pipe(ialu_reg_reg_shift);
10467 %}
10468
10469 // This pattern is automatically generated from aarch64_ad.m4
10470 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10471 // val | (-1 ^ (val >>> shift)) ==> orn
10472 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10473 iRegL src1, iRegL src2,
10474 immI src3, immL_M1 src4) %{
10475 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10476 ins_cost(1.9 * INSN_COST);
10477 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10478
10479 ins_encode %{
10480 __ orn(as_Register($dst$$reg),
10481 as_Register($src1$$reg),
10482 as_Register($src2$$reg),
10483 Assembler::LSR,
10484 $src3$$constant & 0x3f);
10485 %}
10486
10487 ins_pipe(ialu_reg_reg_shift);
10488 %}
10489
10490 // This pattern is automatically generated from aarch64_ad.m4
10491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10492 // val | (-1 ^ (val >> shift)) ==> ornw
10493 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10494 iRegIorL2I src1, iRegIorL2I src2,
10495 immI src3, immI_M1 src4) %{
10496 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10497 ins_cost(1.9 * INSN_COST);
10498 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10499
10500 ins_encode %{
10501 __ ornw(as_Register($dst$$reg),
10502 as_Register($src1$$reg),
10503 as_Register($src2$$reg),
10504 Assembler::ASR,
10505 $src3$$constant & 0x1f);
10506 %}
10507
10508 ins_pipe(ialu_reg_reg_shift);
10509 %}
10510
10511 // This pattern is automatically generated from aarch64_ad.m4
10512 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10513 // val | (-1 ^ (val >> shift)) ==> orn
10514 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10515 iRegL src1, iRegL src2,
10516 immI src3, immL_M1 src4) %{
10517 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10518 ins_cost(1.9 * INSN_COST);
10519 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10520
10521 ins_encode %{
10522 __ orn(as_Register($dst$$reg),
10523 as_Register($src1$$reg),
10524 as_Register($src2$$reg),
10525 Assembler::ASR,
10526 $src3$$constant & 0x3f);
10527 %}
10528
10529 ins_pipe(ialu_reg_reg_shift);
10530 %}
10531
10532 // This pattern is automatically generated from aarch64_ad.m4
10533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10534 // val | (-1 ^ (val ror shift)) ==> ornw
10535 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10536 iRegIorL2I src1, iRegIorL2I src2,
10537 immI src3, immI_M1 src4) %{
10538 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10539 ins_cost(1.9 * INSN_COST);
10540 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10541
10542 ins_encode %{
10543 __ ornw(as_Register($dst$$reg),
10544 as_Register($src1$$reg),
10545 as_Register($src2$$reg),
10546 Assembler::ROR,
10547 $src3$$constant & 0x1f);
10548 %}
10549
10550 ins_pipe(ialu_reg_reg_shift);
10551 %}
10552
10553 // This pattern is automatically generated from aarch64_ad.m4
10554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10555 // val | (-1 ^ (val ror shift)) ==> orn
10556 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10557 iRegL src1, iRegL src2,
10558 immI src3, immL_M1 src4) %{
10559 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10560 ins_cost(1.9 * INSN_COST);
10561 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10562
10563 ins_encode %{
10564 __ orn(as_Register($dst$$reg),
10565 as_Register($src1$$reg),
10566 as_Register($src2$$reg),
10567 Assembler::ROR,
10568 $src3$$constant & 0x3f);
10569 %}
10570
10571 ins_pipe(ialu_reg_reg_shift);
10572 %}
10573
10574 // This pattern is automatically generated from aarch64_ad.m4
10575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10576 // val | (-1 ^ (val << shift)) ==> ornw
10577 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10578 iRegIorL2I src1, iRegIorL2I src2,
10579 immI src3, immI_M1 src4) %{
10580 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10581 ins_cost(1.9 * INSN_COST);
10582 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10583
10584 ins_encode %{
10585 __ ornw(as_Register($dst$$reg),
10586 as_Register($src1$$reg),
10587 as_Register($src2$$reg),
10588 Assembler::LSL,
10589 $src3$$constant & 0x1f);
10590 %}
10591
10592 ins_pipe(ialu_reg_reg_shift);
10593 %}
10594
10595 // This pattern is automatically generated from aarch64_ad.m4
10596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10597 // val | (-1 ^ (val << shift)) ==> orn
10598 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10599 iRegL src1, iRegL src2,
10600 immI src3, immL_M1 src4) %{
10601 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10602 ins_cost(1.9 * INSN_COST);
10603 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10604
10605 ins_encode %{
10606 __ orn(as_Register($dst$$reg),
10607 as_Register($src1$$reg),
10608 as_Register($src2$$reg),
10609 Assembler::LSL,
10610 $src3$$constant & 0x3f);
10611 %}
10612
10613 ins_pipe(ialu_reg_reg_shift);
10614 %}
10615
10616 // This pattern is automatically generated from aarch64_ad.m4
10617 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10618 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10619 iRegIorL2I src1, iRegIorL2I src2,
10620 immI src3) %{
10621 match(Set dst (AndI src1 (URShiftI src2 src3)));
10622
10623 ins_cost(1.9 * INSN_COST);
10624 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10625
10626 ins_encode %{
10627 __ andw(as_Register($dst$$reg),
10628 as_Register($src1$$reg),
10629 as_Register($src2$$reg),
10630 Assembler::LSR,
10631 $src3$$constant & 0x1f);
10632 %}
10633
10634 ins_pipe(ialu_reg_reg_shift);
10635 %}
10636
10637 // This pattern is automatically generated from aarch64_ad.m4
10638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10639 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10640 iRegL src1, iRegL src2,
10641 immI src3) %{
10642 match(Set dst (AndL src1 (URShiftL src2 src3)));
10643
10644 ins_cost(1.9 * INSN_COST);
10645 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10646
10647 ins_encode %{
10648 __ andr(as_Register($dst$$reg),
10649 as_Register($src1$$reg),
10650 as_Register($src2$$reg),
10651 Assembler::LSR,
10652 $src3$$constant & 0x3f);
10653 %}
10654
10655 ins_pipe(ialu_reg_reg_shift);
10656 %}
10657
10658 // This pattern is automatically generated from aarch64_ad.m4
10659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10660 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10661 iRegIorL2I src1, iRegIorL2I src2,
10662 immI src3) %{
10663 match(Set dst (AndI src1 (RShiftI src2 src3)));
10664
10665 ins_cost(1.9 * INSN_COST);
10666 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10667
10668 ins_encode %{
10669 __ andw(as_Register($dst$$reg),
10670 as_Register($src1$$reg),
10671 as_Register($src2$$reg),
10672 Assembler::ASR,
10673 $src3$$constant & 0x1f);
10674 %}
10675
10676 ins_pipe(ialu_reg_reg_shift);
10677 %}
10678
10679 // This pattern is automatically generated from aarch64_ad.m4
10680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10681 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10682 iRegL src1, iRegL src2,
10683 immI src3) %{
10684 match(Set dst (AndL src1 (RShiftL src2 src3)));
10685
10686 ins_cost(1.9 * INSN_COST);
10687 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10688
10689 ins_encode %{
10690 __ andr(as_Register($dst$$reg),
10691 as_Register($src1$$reg),
10692 as_Register($src2$$reg),
10693 Assembler::ASR,
10694 $src3$$constant & 0x3f);
10695 %}
10696
10697 ins_pipe(ialu_reg_reg_shift);
10698 %}
10699
10700 // This pattern is automatically generated from aarch64_ad.m4
10701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10702 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10703 iRegIorL2I src1, iRegIorL2I src2,
10704 immI src3) %{
10705 match(Set dst (AndI src1 (LShiftI src2 src3)));
10706
10707 ins_cost(1.9 * INSN_COST);
10708 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10709
10710 ins_encode %{
10711 __ andw(as_Register($dst$$reg),
10712 as_Register($src1$$reg),
10713 as_Register($src2$$reg),
10714 Assembler::LSL,
10715 $src3$$constant & 0x1f);
10716 %}
10717
10718 ins_pipe(ialu_reg_reg_shift);
10719 %}
10720
10721 // This pattern is automatically generated from aarch64_ad.m4
10722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10723 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10724 iRegL src1, iRegL src2,
10725 immI src3) %{
10726 match(Set dst (AndL src1 (LShiftL src2 src3)));
10727
10728 ins_cost(1.9 * INSN_COST);
10729 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10730
10731 ins_encode %{
10732 __ andr(as_Register($dst$$reg),
10733 as_Register($src1$$reg),
10734 as_Register($src2$$reg),
10735 Assembler::LSL,
10736 $src3$$constant & 0x3f);
10737 %}
10738
10739 ins_pipe(ialu_reg_reg_shift);
10740 %}
10741
10742 // This pattern is automatically generated from aarch64_ad.m4
10743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10744 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10745 iRegIorL2I src1, iRegIorL2I src2,
10746 immI src3) %{
10747 match(Set dst (AndI src1 (RotateRight src2 src3)));
10748
10749 ins_cost(1.9 * INSN_COST);
10750 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10751
10752 ins_encode %{
10753 __ andw(as_Register($dst$$reg),
10754 as_Register($src1$$reg),
10755 as_Register($src2$$reg),
10756 Assembler::ROR,
10757 $src3$$constant & 0x1f);
10758 %}
10759
10760 ins_pipe(ialu_reg_reg_shift);
10761 %}
10762
10763 // This pattern is automatically generated from aarch64_ad.m4
10764 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10765 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10766 iRegL src1, iRegL src2,
10767 immI src3) %{
10768 match(Set dst (AndL src1 (RotateRight src2 src3)));
10769
10770 ins_cost(1.9 * INSN_COST);
10771 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10772
10773 ins_encode %{
10774 __ andr(as_Register($dst$$reg),
10775 as_Register($src1$$reg),
10776 as_Register($src2$$reg),
10777 Assembler::ROR,
10778 $src3$$constant & 0x3f);
10779 %}
10780
10781 ins_pipe(ialu_reg_reg_shift);
10782 %}
10783
10784 // This pattern is automatically generated from aarch64_ad.m4
10785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10786 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10787 iRegIorL2I src1, iRegIorL2I src2,
10788 immI src3) %{
10789 match(Set dst (XorI src1 (URShiftI src2 src3)));
10790
10791 ins_cost(1.9 * INSN_COST);
10792 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10793
10794 ins_encode %{
10795 __ eorw(as_Register($dst$$reg),
10796 as_Register($src1$$reg),
10797 as_Register($src2$$reg),
10798 Assembler::LSR,
10799 $src3$$constant & 0x1f);
10800 %}
10801
10802 ins_pipe(ialu_reg_reg_shift);
10803 %}
10804
10805 // This pattern is automatically generated from aarch64_ad.m4
10806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10807 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10808 iRegL src1, iRegL src2,
10809 immI src3) %{
10810 match(Set dst (XorL src1 (URShiftL src2 src3)));
10811
10812 ins_cost(1.9 * INSN_COST);
10813 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10814
10815 ins_encode %{
10816 __ eor(as_Register($dst$$reg),
10817 as_Register($src1$$reg),
10818 as_Register($src2$$reg),
10819 Assembler::LSR,
10820 $src3$$constant & 0x3f);
10821 %}
10822
10823 ins_pipe(ialu_reg_reg_shift);
10824 %}
10825
10826 // This pattern is automatically generated from aarch64_ad.m4
10827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10828 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10829 iRegIorL2I src1, iRegIorL2I src2,
10830 immI src3) %{
10831 match(Set dst (XorI src1 (RShiftI src2 src3)));
10832
10833 ins_cost(1.9 * INSN_COST);
10834 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10835
10836 ins_encode %{
10837 __ eorw(as_Register($dst$$reg),
10838 as_Register($src1$$reg),
10839 as_Register($src2$$reg),
10840 Assembler::ASR,
10841 $src3$$constant & 0x1f);
10842 %}
10843
10844 ins_pipe(ialu_reg_reg_shift);
10845 %}
10846
10847 // This pattern is automatically generated from aarch64_ad.m4
10848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10849 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10850 iRegL src1, iRegL src2,
10851 immI src3) %{
10852 match(Set dst (XorL src1 (RShiftL src2 src3)));
10853
10854 ins_cost(1.9 * INSN_COST);
10855 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10856
10857 ins_encode %{
10858 __ eor(as_Register($dst$$reg),
10859 as_Register($src1$$reg),
10860 as_Register($src2$$reg),
10861 Assembler::ASR,
10862 $src3$$constant & 0x3f);
10863 %}
10864
10865 ins_pipe(ialu_reg_reg_shift);
10866 %}
10867
10868 // This pattern is automatically generated from aarch64_ad.m4
10869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10870 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10871 iRegIorL2I src1, iRegIorL2I src2,
10872 immI src3) %{
10873 match(Set dst (XorI src1 (LShiftI src2 src3)));
10874
10875 ins_cost(1.9 * INSN_COST);
10876 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10877
10878 ins_encode %{
10879 __ eorw(as_Register($dst$$reg),
10880 as_Register($src1$$reg),
10881 as_Register($src2$$reg),
10882 Assembler::LSL,
10883 $src3$$constant & 0x1f);
10884 %}
10885
10886 ins_pipe(ialu_reg_reg_shift);
10887 %}
10888
10889 // This pattern is automatically generated from aarch64_ad.m4
10890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10891 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10892 iRegL src1, iRegL src2,
10893 immI src3) %{
10894 match(Set dst (XorL src1 (LShiftL src2 src3)));
10895
10896 ins_cost(1.9 * INSN_COST);
10897 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10898
10899 ins_encode %{
10900 __ eor(as_Register($dst$$reg),
10901 as_Register($src1$$reg),
10902 as_Register($src2$$reg),
10903 Assembler::LSL,
10904 $src3$$constant & 0x3f);
10905 %}
10906
10907 ins_pipe(ialu_reg_reg_shift);
10908 %}
10909
10910 // This pattern is automatically generated from aarch64_ad.m4
10911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10912 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10913 iRegIorL2I src1, iRegIorL2I src2,
10914 immI src3) %{
10915 match(Set dst (XorI src1 (RotateRight src2 src3)));
10916
10917 ins_cost(1.9 * INSN_COST);
10918 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10919
10920 ins_encode %{
10921 __ eorw(as_Register($dst$$reg),
10922 as_Register($src1$$reg),
10923 as_Register($src2$$reg),
10924 Assembler::ROR,
10925 $src3$$constant & 0x1f);
10926 %}
10927
10928 ins_pipe(ialu_reg_reg_shift);
10929 %}
10930
10931 // This pattern is automatically generated from aarch64_ad.m4
10932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10933 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10934 iRegL src1, iRegL src2,
10935 immI src3) %{
10936 match(Set dst (XorL src1 (RotateRight src2 src3)));
10937
10938 ins_cost(1.9 * INSN_COST);
10939 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10940
10941 ins_encode %{
10942 __ eor(as_Register($dst$$reg),
10943 as_Register($src1$$reg),
10944 as_Register($src2$$reg),
10945 Assembler::ROR,
10946 $src3$$constant & 0x3f);
10947 %}
10948
10949 ins_pipe(ialu_reg_reg_shift);
10950 %}
10951
10952 // This pattern is automatically generated from aarch64_ad.m4
10953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10954 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10955 iRegIorL2I src1, iRegIorL2I src2,
10956 immI src3) %{
10957 match(Set dst (OrI src1 (URShiftI src2 src3)));
10958
10959 ins_cost(1.9 * INSN_COST);
10960 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10961
10962 ins_encode %{
10963 __ orrw(as_Register($dst$$reg),
10964 as_Register($src1$$reg),
10965 as_Register($src2$$reg),
10966 Assembler::LSR,
10967 $src3$$constant & 0x1f);
10968 %}
10969
10970 ins_pipe(ialu_reg_reg_shift);
10971 %}
10972
10973 // This pattern is automatically generated from aarch64_ad.m4
10974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10975 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
10976 iRegL src1, iRegL src2,
10977 immI src3) %{
10978 match(Set dst (OrL src1 (URShiftL src2 src3)));
10979
10980 ins_cost(1.9 * INSN_COST);
10981 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
10982
10983 ins_encode %{
10984 __ orr(as_Register($dst$$reg),
10985 as_Register($src1$$reg),
10986 as_Register($src2$$reg),
10987 Assembler::LSR,
10988 $src3$$constant & 0x3f);
10989 %}
10990
10991 ins_pipe(ialu_reg_reg_shift);
10992 %}
10993
10994 // This pattern is automatically generated from aarch64_ad.m4
10995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10996 instruct OrI_reg_RShift_reg(iRegINoSp dst,
10997 iRegIorL2I src1, iRegIorL2I src2,
10998 immI src3) %{
10999 match(Set dst (OrI src1 (RShiftI src2 src3)));
11000
11001 ins_cost(1.9 * INSN_COST);
11002 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11003
11004 ins_encode %{
11005 __ orrw(as_Register($dst$$reg),
11006 as_Register($src1$$reg),
11007 as_Register($src2$$reg),
11008 Assembler::ASR,
11009 $src3$$constant & 0x1f);
11010 %}
11011
11012 ins_pipe(ialu_reg_reg_shift);
11013 %}
11014
11015 // This pattern is automatically generated from aarch64_ad.m4
11016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11017 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11018 iRegL src1, iRegL src2,
11019 immI src3) %{
11020 match(Set dst (OrL src1 (RShiftL src2 src3)));
11021
11022 ins_cost(1.9 * INSN_COST);
11023 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11024
11025 ins_encode %{
11026 __ orr(as_Register($dst$$reg),
11027 as_Register($src1$$reg),
11028 as_Register($src2$$reg),
11029 Assembler::ASR,
11030 $src3$$constant & 0x3f);
11031 %}
11032
11033 ins_pipe(ialu_reg_reg_shift);
11034 %}
11035
11036 // This pattern is automatically generated from aarch64_ad.m4
11037 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11038 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11039 iRegIorL2I src1, iRegIorL2I src2,
11040 immI src3) %{
11041 match(Set dst (OrI src1 (LShiftI src2 src3)));
11042
11043 ins_cost(1.9 * INSN_COST);
11044 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11045
11046 ins_encode %{
11047 __ orrw(as_Register($dst$$reg),
11048 as_Register($src1$$reg),
11049 as_Register($src2$$reg),
11050 Assembler::LSL,
11051 $src3$$constant & 0x1f);
11052 %}
11053
11054 ins_pipe(ialu_reg_reg_shift);
11055 %}
11056
11057 // This pattern is automatically generated from aarch64_ad.m4
11058 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11059 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11060 iRegL src1, iRegL src2,
11061 immI src3) %{
11062 match(Set dst (OrL src1 (LShiftL src2 src3)));
11063
11064 ins_cost(1.9 * INSN_COST);
11065 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11066
11067 ins_encode %{
11068 __ orr(as_Register($dst$$reg),
11069 as_Register($src1$$reg),
11070 as_Register($src2$$reg),
11071 Assembler::LSL,
11072 $src3$$constant & 0x3f);
11073 %}
11074
11075 ins_pipe(ialu_reg_reg_shift);
11076 %}
11077
11078 // This pattern is automatically generated from aarch64_ad.m4
11079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11080 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11081 iRegIorL2I src1, iRegIorL2I src2,
11082 immI src3) %{
11083 match(Set dst (OrI src1 (RotateRight src2 src3)));
11084
11085 ins_cost(1.9 * INSN_COST);
11086 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11087
11088 ins_encode %{
11089 __ orrw(as_Register($dst$$reg),
11090 as_Register($src1$$reg),
11091 as_Register($src2$$reg),
11092 Assembler::ROR,
11093 $src3$$constant & 0x1f);
11094 %}
11095
11096 ins_pipe(ialu_reg_reg_shift);
11097 %}
11098
11099 // This pattern is automatically generated from aarch64_ad.m4
11100 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11101 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11102 iRegL src1, iRegL src2,
11103 immI src3) %{
11104 match(Set dst (OrL src1 (RotateRight src2 src3)));
11105
11106 ins_cost(1.9 * INSN_COST);
11107 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11108
11109 ins_encode %{
11110 __ orr(as_Register($dst$$reg),
11111 as_Register($src1$$reg),
11112 as_Register($src2$$reg),
11113 Assembler::ROR,
11114 $src3$$constant & 0x3f);
11115 %}
11116
11117 ins_pipe(ialu_reg_reg_shift);
11118 %}
11119
11120 // This pattern is automatically generated from aarch64_ad.m4
11121 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11122 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11123 iRegIorL2I src1, iRegIorL2I src2,
11124 immI src3) %{
11125 match(Set dst (AddI src1 (URShiftI src2 src3)));
11126
11127 ins_cost(1.9 * INSN_COST);
11128 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11129
11130 ins_encode %{
11131 __ addw(as_Register($dst$$reg),
11132 as_Register($src1$$reg),
11133 as_Register($src2$$reg),
11134 Assembler::LSR,
11135 $src3$$constant & 0x1f);
11136 %}
11137
11138 ins_pipe(ialu_reg_reg_shift);
11139 %}
11140
11141 // This pattern is automatically generated from aarch64_ad.m4
11142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11143 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11144 iRegL src1, iRegL src2,
11145 immI src3) %{
11146 match(Set dst (AddL src1 (URShiftL src2 src3)));
11147
11148 ins_cost(1.9 * INSN_COST);
11149 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11150
11151 ins_encode %{
11152 __ add(as_Register($dst$$reg),
11153 as_Register($src1$$reg),
11154 as_Register($src2$$reg),
11155 Assembler::LSR,
11156 $src3$$constant & 0x3f);
11157 %}
11158
11159 ins_pipe(ialu_reg_reg_shift);
11160 %}
11161
11162 // This pattern is automatically generated from aarch64_ad.m4
11163 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11164 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11165 iRegIorL2I src1, iRegIorL2I src2,
11166 immI src3) %{
11167 match(Set dst (AddI src1 (RShiftI src2 src3)));
11168
11169 ins_cost(1.9 * INSN_COST);
11170 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11171
11172 ins_encode %{
11173 __ addw(as_Register($dst$$reg),
11174 as_Register($src1$$reg),
11175 as_Register($src2$$reg),
11176 Assembler::ASR,
11177 $src3$$constant & 0x1f);
11178 %}
11179
11180 ins_pipe(ialu_reg_reg_shift);
11181 %}
11182
11183 // This pattern is automatically generated from aarch64_ad.m4
11184 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11185 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11186 iRegL src1, iRegL src2,
11187 immI src3) %{
11188 match(Set dst (AddL src1 (RShiftL src2 src3)));
11189
11190 ins_cost(1.9 * INSN_COST);
11191 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11192
11193 ins_encode %{
11194 __ add(as_Register($dst$$reg),
11195 as_Register($src1$$reg),
11196 as_Register($src2$$reg),
11197 Assembler::ASR,
11198 $src3$$constant & 0x3f);
11199 %}
11200
11201 ins_pipe(ialu_reg_reg_shift);
11202 %}
11203
11204 // This pattern is automatically generated from aarch64_ad.m4
11205 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11206 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11207 iRegIorL2I src1, iRegIorL2I src2,
11208 immI src3) %{
11209 match(Set dst (AddI src1 (LShiftI src2 src3)));
11210
11211 ins_cost(1.9 * INSN_COST);
11212 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11213
11214 ins_encode %{
11215 __ addw(as_Register($dst$$reg),
11216 as_Register($src1$$reg),
11217 as_Register($src2$$reg),
11218 Assembler::LSL,
11219 $src3$$constant & 0x1f);
11220 %}
11221
11222 ins_pipe(ialu_reg_reg_shift);
11223 %}
11224
11225 // This pattern is automatically generated from aarch64_ad.m4
11226 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11227 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11228 iRegL src1, iRegL src2,
11229 immI src3) %{
11230 match(Set dst (AddL src1 (LShiftL src2 src3)));
11231
11232 ins_cost(1.9 * INSN_COST);
11233 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11234
11235 ins_encode %{
11236 __ add(as_Register($dst$$reg),
11237 as_Register($src1$$reg),
11238 as_Register($src2$$reg),
11239 Assembler::LSL,
11240 $src3$$constant & 0x3f);
11241 %}
11242
11243 ins_pipe(ialu_reg_reg_shift);
11244 %}
11245
11246 // This pattern is automatically generated from aarch64_ad.m4
11247 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11248 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11249 iRegIorL2I src1, iRegIorL2I src2,
11250 immI src3) %{
11251 match(Set dst (SubI src1 (URShiftI src2 src3)));
11252
11253 ins_cost(1.9 * INSN_COST);
11254 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11255
11256 ins_encode %{
11257 __ subw(as_Register($dst$$reg),
11258 as_Register($src1$$reg),
11259 as_Register($src2$$reg),
11260 Assembler::LSR,
11261 $src3$$constant & 0x1f);
11262 %}
11263
11264 ins_pipe(ialu_reg_reg_shift);
11265 %}
11266
11267 // This pattern is automatically generated from aarch64_ad.m4
11268 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11269 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11270 iRegL src1, iRegL src2,
11271 immI src3) %{
11272 match(Set dst (SubL src1 (URShiftL src2 src3)));
11273
11274 ins_cost(1.9 * INSN_COST);
11275 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11276
11277 ins_encode %{
11278 __ sub(as_Register($dst$$reg),
11279 as_Register($src1$$reg),
11280 as_Register($src2$$reg),
11281 Assembler::LSR,
11282 $src3$$constant & 0x3f);
11283 %}
11284
11285 ins_pipe(ialu_reg_reg_shift);
11286 %}
11287
11288 // This pattern is automatically generated from aarch64_ad.m4
11289 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11290 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11291 iRegIorL2I src1, iRegIorL2I src2,
11292 immI src3) %{
11293 match(Set dst (SubI src1 (RShiftI src2 src3)));
11294
11295 ins_cost(1.9 * INSN_COST);
11296 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11297
11298 ins_encode %{
11299 __ subw(as_Register($dst$$reg),
11300 as_Register($src1$$reg),
11301 as_Register($src2$$reg),
11302 Assembler::ASR,
11303 $src3$$constant & 0x1f);
11304 %}
11305
11306 ins_pipe(ialu_reg_reg_shift);
11307 %}
11308
11309 // This pattern is automatically generated from aarch64_ad.m4
11310 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11311 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11312 iRegL src1, iRegL src2,
11313 immI src3) %{
11314 match(Set dst (SubL src1 (RShiftL src2 src3)));
11315
11316 ins_cost(1.9 * INSN_COST);
11317 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11318
11319 ins_encode %{
11320 __ sub(as_Register($dst$$reg),
11321 as_Register($src1$$reg),
11322 as_Register($src2$$reg),
11323 Assembler::ASR,
11324 $src3$$constant & 0x3f);
11325 %}
11326
11327 ins_pipe(ialu_reg_reg_shift);
11328 %}
11329
11330 // This pattern is automatically generated from aarch64_ad.m4
11331 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11332 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11333 iRegIorL2I src1, iRegIorL2I src2,
11334 immI src3) %{
11335 match(Set dst (SubI src1 (LShiftI src2 src3)));
11336
11337 ins_cost(1.9 * INSN_COST);
11338 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11339
11340 ins_encode %{
11341 __ subw(as_Register($dst$$reg),
11342 as_Register($src1$$reg),
11343 as_Register($src2$$reg),
11344 Assembler::LSL,
11345 $src3$$constant & 0x1f);
11346 %}
11347
11348 ins_pipe(ialu_reg_reg_shift);
11349 %}
11350
11351 // This pattern is automatically generated from aarch64_ad.m4
11352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11353 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11354 iRegL src1, iRegL src2,
11355 immI src3) %{
11356 match(Set dst (SubL src1 (LShiftL src2 src3)));
11357
11358 ins_cost(1.9 * INSN_COST);
11359 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11360
11361 ins_encode %{
11362 __ sub(as_Register($dst$$reg),
11363 as_Register($src1$$reg),
11364 as_Register($src2$$reg),
11365 Assembler::LSL,
11366 $src3$$constant & 0x3f);
11367 %}
11368
11369 ins_pipe(ialu_reg_reg_shift);
11370 %}
11371
11372 // This pattern is automatically generated from aarch64_ad.m4
11373 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11374
11375 // Shift Left followed by Shift Right.
11376 // This idiom is used by the compiler for the i2b bytecode etc.
11377 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11378 %{
11379 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11380 ins_cost(INSN_COST * 2);
11381 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11382 ins_encode %{
11383 int lshift = $lshift_count$$constant & 63;
11384 int rshift = $rshift_count$$constant & 63;
11385 int s = 63 - lshift;
11386 int r = (rshift - lshift) & 63;
11387 __ sbfm(as_Register($dst$$reg),
11388 as_Register($src$$reg),
11389 r, s);
11390 %}
11391
11392 ins_pipe(ialu_reg_shift);
11393 %}
11394
11395 // This pattern is automatically generated from aarch64_ad.m4
11396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11397
11398 // Shift Left followed by Shift Right.
11399 // This idiom is used by the compiler for the i2b bytecode etc.
11400 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11401 %{
11402 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11403 ins_cost(INSN_COST * 2);
11404 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11405 ins_encode %{
11406 int lshift = $lshift_count$$constant & 31;
11407 int rshift = $rshift_count$$constant & 31;
11408 int s = 31 - lshift;
11409 int r = (rshift - lshift) & 31;
11410 __ sbfmw(as_Register($dst$$reg),
11411 as_Register($src$$reg),
11412 r, s);
11413 %}
11414
11415 ins_pipe(ialu_reg_shift);
11416 %}
11417
11418 // This pattern is automatically generated from aarch64_ad.m4
11419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11420
11421 // Shift Left followed by Shift Right.
11422 // This idiom is used by the compiler for the i2b bytecode etc.
11423 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11424 %{
11425 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11426 ins_cost(INSN_COST * 2);
11427 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11428 ins_encode %{
11429 int lshift = $lshift_count$$constant & 63;
11430 int rshift = $rshift_count$$constant & 63;
11431 int s = 63 - lshift;
11432 int r = (rshift - lshift) & 63;
11433 __ ubfm(as_Register($dst$$reg),
11434 as_Register($src$$reg),
11435 r, s);
11436 %}
11437
11438 ins_pipe(ialu_reg_shift);
11439 %}
11440
11441 // This pattern is automatically generated from aarch64_ad.m4
11442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11443
11444 // Shift Left followed by Shift Right.
11445 // This idiom is used by the compiler for the i2b bytecode etc.
11446 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11447 %{
11448 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11449 ins_cost(INSN_COST * 2);
11450 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11451 ins_encode %{
11452 int lshift = $lshift_count$$constant & 31;
11453 int rshift = $rshift_count$$constant & 31;
11454 int s = 31 - lshift;
11455 int r = (rshift - lshift) & 31;
11456 __ ubfmw(as_Register($dst$$reg),
11457 as_Register($src$$reg),
11458 r, s);
11459 %}
11460
11461 ins_pipe(ialu_reg_shift);
11462 %}
11463
11464 // Bitfield extract with shift & mask
11465
11466 // This pattern is automatically generated from aarch64_ad.m4
11467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11468 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11469 %{
11470 match(Set dst (AndI (URShiftI src rshift) mask));
11471 // Make sure we are not going to exceed what ubfxw can do.
11472 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11473
11474 ins_cost(INSN_COST);
11475 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11476 ins_encode %{
11477 int rshift = $rshift$$constant & 31;
11478 intptr_t mask = $mask$$constant;
11479 int width = exact_log2(mask+1);
11480 __ ubfxw(as_Register($dst$$reg),
11481 as_Register($src$$reg), rshift, width);
11482 %}
11483 ins_pipe(ialu_reg_shift);
11484 %}
11485
11486 // This pattern is automatically generated from aarch64_ad.m4
11487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11488 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11489 %{
11490 match(Set dst (AndL (URShiftL src rshift) mask));
11491 // Make sure we are not going to exceed what ubfx can do.
11492 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11493
11494 ins_cost(INSN_COST);
11495 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11496 ins_encode %{
11497 int rshift = $rshift$$constant & 63;
11498 intptr_t mask = $mask$$constant;
11499 int width = exact_log2_long(mask+1);
11500 __ ubfx(as_Register($dst$$reg),
11501 as_Register($src$$reg), rshift, width);
11502 %}
11503 ins_pipe(ialu_reg_shift);
11504 %}
11505
11506
11507 // This pattern is automatically generated from aarch64_ad.m4
11508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11509
11510 // We can use ubfx when extending an And with a mask when we know mask
11511 // is positive. We know that because immI_bitmask guarantees it.
11512 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11513 %{
11514 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11515 // Make sure we are not going to exceed what ubfxw can do.
11516 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11517
11518 ins_cost(INSN_COST * 2);
11519 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11520 ins_encode %{
11521 int rshift = $rshift$$constant & 31;
11522 intptr_t mask = $mask$$constant;
11523 int width = exact_log2(mask+1);
11524 __ ubfx(as_Register($dst$$reg),
11525 as_Register($src$$reg), rshift, width);
11526 %}
11527 ins_pipe(ialu_reg_shift);
11528 %}
11529
11530
11531 // This pattern is automatically generated from aarch64_ad.m4
11532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11533
11534 // We can use ubfiz when masking by a positive number and then left shifting the result.
11535 // We know that the mask is positive because immI_bitmask guarantees it.
11536 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11537 %{
11538 match(Set dst (LShiftI (AndI src mask) lshift));
11539 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11540
11541 ins_cost(INSN_COST);
11542 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11543 ins_encode %{
11544 int lshift = $lshift$$constant & 31;
11545 intptr_t mask = $mask$$constant;
11546 int width = exact_log2(mask+1);
11547 __ ubfizw(as_Register($dst$$reg),
11548 as_Register($src$$reg), lshift, width);
11549 %}
11550 ins_pipe(ialu_reg_shift);
11551 %}
11552
11553 // This pattern is automatically generated from aarch64_ad.m4
11554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11555
11556 // We can use ubfiz when masking by a positive number and then left shifting the result.
11557 // We know that the mask is positive because immL_bitmask guarantees it.
11558 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11559 %{
11560 match(Set dst (LShiftL (AndL src mask) lshift));
11561 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11562
11563 ins_cost(INSN_COST);
11564 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11565 ins_encode %{
11566 int lshift = $lshift$$constant & 63;
11567 intptr_t mask = $mask$$constant;
11568 int width = exact_log2_long(mask+1);
11569 __ ubfiz(as_Register($dst$$reg),
11570 as_Register($src$$reg), lshift, width);
11571 %}
11572 ins_pipe(ialu_reg_shift);
11573 %}
11574
11575 // This pattern is automatically generated from aarch64_ad.m4
11576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11577
11578 // We can use ubfiz when masking by a positive number and then left shifting the result.
11579 // We know that the mask is positive because immI_bitmask guarantees it.
11580 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11581 %{
11582 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11583 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11584
11585 ins_cost(INSN_COST);
11586 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11587 ins_encode %{
11588 int lshift = $lshift$$constant & 31;
11589 intptr_t mask = $mask$$constant;
11590 int width = exact_log2(mask+1);
11591 __ ubfizw(as_Register($dst$$reg),
11592 as_Register($src$$reg), lshift, width);
11593 %}
11594 ins_pipe(ialu_reg_shift);
11595 %}
11596
11597 // This pattern is automatically generated from aarch64_ad.m4
11598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11599
11600 // We can use ubfiz when masking by a positive number and then left shifting the result.
11601 // We know that the mask is positive because immL_bitmask guarantees it.
11602 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11603 %{
11604 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11605 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11606
11607 ins_cost(INSN_COST);
11608 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11609 ins_encode %{
11610 int lshift = $lshift$$constant & 63;
11611 intptr_t mask = $mask$$constant;
11612 int width = exact_log2_long(mask+1);
11613 __ ubfiz(as_Register($dst$$reg),
11614 as_Register($src$$reg), lshift, width);
11615 %}
11616 ins_pipe(ialu_reg_shift);
11617 %}
11618
11619
11620 // This pattern is automatically generated from aarch64_ad.m4
11621 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11622
11623 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11624 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11625 %{
11626 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11627 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11628
11629 ins_cost(INSN_COST);
11630 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11631 ins_encode %{
11632 int lshift = $lshift$$constant & 63;
11633 intptr_t mask = $mask$$constant;
11634 int width = exact_log2(mask+1);
11635 __ ubfiz(as_Register($dst$$reg),
11636 as_Register($src$$reg), lshift, width);
11637 %}
11638 ins_pipe(ialu_reg_shift);
11639 %}
11640
11641 // This pattern is automatically generated from aarch64_ad.m4
11642 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11643
11644 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11645 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11646 %{
11647 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11648 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11649
11650 ins_cost(INSN_COST);
11651 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11652 ins_encode %{
11653 int lshift = $lshift$$constant & 31;
11654 intptr_t mask = $mask$$constant;
11655 int width = exact_log2(mask+1);
11656 __ ubfiz(as_Register($dst$$reg),
11657 as_Register($src$$reg), lshift, width);
11658 %}
11659 ins_pipe(ialu_reg_shift);
11660 %}
11661
11662 // This pattern is automatically generated from aarch64_ad.m4
11663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11664
11665 // Can skip int2long conversions after AND with small bitmask
11666 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11667 %{
11668 match(Set dst (ConvI2L (AndI src msk)));
11669 ins_cost(INSN_COST);
11670 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11671 ins_encode %{
11672 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11673 %}
11674 ins_pipe(ialu_reg_shift);
11675 %}
11676
11677
11678 // Rotations
11679
11680 // This pattern is automatically generated from aarch64_ad.m4
11681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11682 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11683 %{
11684 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11685 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11686
11687 ins_cost(INSN_COST);
11688 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11689
11690 ins_encode %{
11691 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11692 $rshift$$constant & 63);
11693 %}
11694 ins_pipe(ialu_reg_reg_extr);
11695 %}
11696
11697
11698 // This pattern is automatically generated from aarch64_ad.m4
11699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11700 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11701 %{
11702 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11703 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11704
11705 ins_cost(INSN_COST);
11706 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11707
11708 ins_encode %{
11709 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11710 $rshift$$constant & 31);
11711 %}
11712 ins_pipe(ialu_reg_reg_extr);
11713 %}
11714
11715
11716 // This pattern is automatically generated from aarch64_ad.m4
11717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11718 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11719 %{
11720 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11721 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11722
11723 ins_cost(INSN_COST);
11724 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11725
11726 ins_encode %{
11727 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11728 $rshift$$constant & 63);
11729 %}
11730 ins_pipe(ialu_reg_reg_extr);
11731 %}
11732
11733
11734 // This pattern is automatically generated from aarch64_ad.m4
11735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11736 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11737 %{
11738 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11739 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11740
11741 ins_cost(INSN_COST);
11742 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11743
11744 ins_encode %{
11745 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11746 $rshift$$constant & 31);
11747 %}
11748 ins_pipe(ialu_reg_reg_extr);
11749 %}
11750
11751 // This pattern is automatically generated from aarch64_ad.m4
11752 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11753 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11754 %{
11755 match(Set dst (RotateRight src shift));
11756
11757 ins_cost(INSN_COST);
11758 format %{ "ror $dst, $src, $shift" %}
11759
11760 ins_encode %{
11761 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11762 $shift$$constant & 0x1f);
11763 %}
11764 ins_pipe(ialu_reg_reg_vshift);
11765 %}
11766
11767 // This pattern is automatically generated from aarch64_ad.m4
11768 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11769 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11770 %{
11771 match(Set dst (RotateRight src shift));
11772
11773 ins_cost(INSN_COST);
11774 format %{ "ror $dst, $src, $shift" %}
11775
11776 ins_encode %{
11777 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11778 $shift$$constant & 0x3f);
11779 %}
11780 ins_pipe(ialu_reg_reg_vshift);
11781 %}
11782
11783 // This pattern is automatically generated from aarch64_ad.m4
11784 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11785 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11786 %{
11787 match(Set dst (RotateRight src shift));
11788
11789 ins_cost(INSN_COST);
11790 format %{ "ror $dst, $src, $shift" %}
11791
11792 ins_encode %{
11793 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11794 %}
11795 ins_pipe(ialu_reg_reg_vshift);
11796 %}
11797
11798 // This pattern is automatically generated from aarch64_ad.m4
11799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11800 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11801 %{
11802 match(Set dst (RotateRight src shift));
11803
11804 ins_cost(INSN_COST);
11805 format %{ "ror $dst, $src, $shift" %}
11806
11807 ins_encode %{
11808 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11809 %}
11810 ins_pipe(ialu_reg_reg_vshift);
11811 %}
11812
11813 // This pattern is automatically generated from aarch64_ad.m4
11814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11815 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11816 %{
11817 match(Set dst (RotateLeft src shift));
11818
11819 ins_cost(INSN_COST);
11820 format %{ "rol $dst, $src, $shift" %}
11821
11822 ins_encode %{
11823 __ subw(rscratch1, zr, as_Register($shift$$reg));
11824 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11825 %}
11826 ins_pipe(ialu_reg_reg_vshift);
11827 %}
11828
11829 // This pattern is automatically generated from aarch64_ad.m4
11830 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11831 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11832 %{
11833 match(Set dst (RotateLeft src shift));
11834
11835 ins_cost(INSN_COST);
11836 format %{ "rol $dst, $src, $shift" %}
11837
11838 ins_encode %{
11839 __ subw(rscratch1, zr, as_Register($shift$$reg));
11840 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11841 %}
11842 ins_pipe(ialu_reg_reg_vshift);
11843 %}
11844
11845
11846 // Add/subtract (extended)
11847
11848 // This pattern is automatically generated from aarch64_ad.m4
11849 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11850 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11851 %{
11852 match(Set dst (AddL src1 (ConvI2L src2)));
11853 ins_cost(INSN_COST);
11854 format %{ "add $dst, $src1, $src2, sxtw" %}
11855
11856 ins_encode %{
11857 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11858 as_Register($src2$$reg), ext::sxtw);
11859 %}
11860 ins_pipe(ialu_reg_reg);
11861 %}
11862
11863 // This pattern is automatically generated from aarch64_ad.m4
11864 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11865 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11866 %{
11867 match(Set dst (SubL src1 (ConvI2L src2)));
11868 ins_cost(INSN_COST);
11869 format %{ "sub $dst, $src1, $src2, sxtw" %}
11870
11871 ins_encode %{
11872 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11873 as_Register($src2$$reg), ext::sxtw);
11874 %}
11875 ins_pipe(ialu_reg_reg);
11876 %}
11877
11878 // This pattern is automatically generated from aarch64_ad.m4
11879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11880 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11881 %{
11882 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11883 ins_cost(INSN_COST);
11884 format %{ "add $dst, $src1, $src2, sxth" %}
11885
11886 ins_encode %{
11887 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11888 as_Register($src2$$reg), ext::sxth);
11889 %}
11890 ins_pipe(ialu_reg_reg);
11891 %}
11892
11893 // This pattern is automatically generated from aarch64_ad.m4
11894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11895 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11896 %{
11897 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11898 ins_cost(INSN_COST);
11899 format %{ "add $dst, $src1, $src2, sxtb" %}
11900
11901 ins_encode %{
11902 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11903 as_Register($src2$$reg), ext::sxtb);
11904 %}
11905 ins_pipe(ialu_reg_reg);
11906 %}
11907
11908 // This pattern is automatically generated from aarch64_ad.m4
11909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11910 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11911 %{
11912 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11913 ins_cost(INSN_COST);
11914 format %{ "add $dst, $src1, $src2, uxtb" %}
11915
11916 ins_encode %{
11917 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11918 as_Register($src2$$reg), ext::uxtb);
11919 %}
11920 ins_pipe(ialu_reg_reg);
11921 %}
11922
11923 // This pattern is automatically generated from aarch64_ad.m4
11924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11925 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11926 %{
11927 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11928 ins_cost(INSN_COST);
11929 format %{ "add $dst, $src1, $src2, sxth" %}
11930
11931 ins_encode %{
11932 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11933 as_Register($src2$$reg), ext::sxth);
11934 %}
11935 ins_pipe(ialu_reg_reg);
11936 %}
11937
11938 // This pattern is automatically generated from aarch64_ad.m4
11939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11940 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11941 %{
11942 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11943 ins_cost(INSN_COST);
11944 format %{ "add $dst, $src1, $src2, sxtw" %}
11945
11946 ins_encode %{
11947 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11948 as_Register($src2$$reg), ext::sxtw);
11949 %}
11950 ins_pipe(ialu_reg_reg);
11951 %}
11952
11953 // This pattern is automatically generated from aarch64_ad.m4
11954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11955 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11956 %{
11957 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11958 ins_cost(INSN_COST);
11959 format %{ "add $dst, $src1, $src2, sxtb" %}
11960
11961 ins_encode %{
11962 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11963 as_Register($src2$$reg), ext::sxtb);
11964 %}
11965 ins_pipe(ialu_reg_reg);
11966 %}
11967
11968 // This pattern is automatically generated from aarch64_ad.m4
11969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11970 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11971 %{
11972 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11973 ins_cost(INSN_COST);
11974 format %{ "add $dst, $src1, $src2, uxtb" %}
11975
11976 ins_encode %{
11977 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11978 as_Register($src2$$reg), ext::uxtb);
11979 %}
11980 ins_pipe(ialu_reg_reg);
11981 %}
11982
11983 // This pattern is automatically generated from aarch64_ad.m4
11984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11985 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11986 %{
11987 match(Set dst (AddI src1 (AndI src2 mask)));
11988 ins_cost(INSN_COST);
11989 format %{ "addw $dst, $src1, $src2, uxtb" %}
11990
11991 ins_encode %{
11992 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11993 as_Register($src2$$reg), ext::uxtb);
11994 %}
11995 ins_pipe(ialu_reg_reg);
11996 %}
11997
11998 // This pattern is automatically generated from aarch64_ad.m4
11999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12000 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12001 %{
12002 match(Set dst (AddI src1 (AndI src2 mask)));
12003 ins_cost(INSN_COST);
12004 format %{ "addw $dst, $src1, $src2, uxth" %}
12005
12006 ins_encode %{
12007 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12008 as_Register($src2$$reg), ext::uxth);
12009 %}
12010 ins_pipe(ialu_reg_reg);
12011 %}
12012
12013 // This pattern is automatically generated from aarch64_ad.m4
12014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12015 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12016 %{
12017 match(Set dst (AddL src1 (AndL src2 mask)));
12018 ins_cost(INSN_COST);
12019 format %{ "add $dst, $src1, $src2, uxtb" %}
12020
12021 ins_encode %{
12022 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12023 as_Register($src2$$reg), ext::uxtb);
12024 %}
12025 ins_pipe(ialu_reg_reg);
12026 %}
12027
12028 // This pattern is automatically generated from aarch64_ad.m4
12029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12030 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12031 %{
12032 match(Set dst (AddL src1 (AndL src2 mask)));
12033 ins_cost(INSN_COST);
12034 format %{ "add $dst, $src1, $src2, uxth" %}
12035
12036 ins_encode %{
12037 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12038 as_Register($src2$$reg), ext::uxth);
12039 %}
12040 ins_pipe(ialu_reg_reg);
12041 %}
12042
12043 // This pattern is automatically generated from aarch64_ad.m4
12044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12045 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12046 %{
12047 match(Set dst (AddL src1 (AndL src2 mask)));
12048 ins_cost(INSN_COST);
12049 format %{ "add $dst, $src1, $src2, uxtw" %}
12050
12051 ins_encode %{
12052 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12053 as_Register($src2$$reg), ext::uxtw);
12054 %}
12055 ins_pipe(ialu_reg_reg);
12056 %}
12057
12058 // This pattern is automatically generated from aarch64_ad.m4
12059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12060 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12061 %{
12062 match(Set dst (SubI src1 (AndI src2 mask)));
12063 ins_cost(INSN_COST);
12064 format %{ "subw $dst, $src1, $src2, uxtb" %}
12065
12066 ins_encode %{
12067 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12068 as_Register($src2$$reg), ext::uxtb);
12069 %}
12070 ins_pipe(ialu_reg_reg);
12071 %}
12072
12073 // This pattern is automatically generated from aarch64_ad.m4
12074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12075 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12076 %{
12077 match(Set dst (SubI src1 (AndI src2 mask)));
12078 ins_cost(INSN_COST);
12079 format %{ "subw $dst, $src1, $src2, uxth" %}
12080
12081 ins_encode %{
12082 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12083 as_Register($src2$$reg), ext::uxth);
12084 %}
12085 ins_pipe(ialu_reg_reg);
12086 %}
12087
12088 // This pattern is automatically generated from aarch64_ad.m4
12089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12090 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12091 %{
12092 match(Set dst (SubL src1 (AndL src2 mask)));
12093 ins_cost(INSN_COST);
12094 format %{ "sub $dst, $src1, $src2, uxtb" %}
12095
12096 ins_encode %{
12097 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12098 as_Register($src2$$reg), ext::uxtb);
12099 %}
12100 ins_pipe(ialu_reg_reg);
12101 %}
12102
12103 // This pattern is automatically generated from aarch64_ad.m4
12104 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12105 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12106 %{
12107 match(Set dst (SubL src1 (AndL src2 mask)));
12108 ins_cost(INSN_COST);
12109 format %{ "sub $dst, $src1, $src2, uxth" %}
12110
12111 ins_encode %{
12112 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12113 as_Register($src2$$reg), ext::uxth);
12114 %}
12115 ins_pipe(ialu_reg_reg);
12116 %}
12117
12118 // This pattern is automatically generated from aarch64_ad.m4
12119 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12120 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12121 %{
12122 match(Set dst (SubL src1 (AndL src2 mask)));
12123 ins_cost(INSN_COST);
12124 format %{ "sub $dst, $src1, $src2, uxtw" %}
12125
12126 ins_encode %{
12127 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12128 as_Register($src2$$reg), ext::uxtw);
12129 %}
12130 ins_pipe(ialu_reg_reg);
12131 %}
12132
12133
12134 // This pattern is automatically generated from aarch64_ad.m4
12135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12136 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12137 %{
12138 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12139 ins_cost(1.9 * INSN_COST);
12140 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12141
12142 ins_encode %{
12143 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12144 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12145 %}
12146 ins_pipe(ialu_reg_reg_shift);
12147 %}
12148
12149 // This pattern is automatically generated from aarch64_ad.m4
12150 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12151 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12152 %{
12153 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12154 ins_cost(1.9 * INSN_COST);
12155 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12156
12157 ins_encode %{
12158 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12159 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12160 %}
12161 ins_pipe(ialu_reg_reg_shift);
12162 %}
12163
12164 // This pattern is automatically generated from aarch64_ad.m4
12165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12166 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12167 %{
12168 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12169 ins_cost(1.9 * INSN_COST);
12170 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12171
12172 ins_encode %{
12173 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12174 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12175 %}
12176 ins_pipe(ialu_reg_reg_shift);
12177 %}
12178
12179 // This pattern is automatically generated from aarch64_ad.m4
12180 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12181 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12182 %{
12183 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12184 ins_cost(1.9 * INSN_COST);
12185 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12186
12187 ins_encode %{
12188 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12189 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12190 %}
12191 ins_pipe(ialu_reg_reg_shift);
12192 %}
12193
12194 // This pattern is automatically generated from aarch64_ad.m4
12195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12196 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12197 %{
12198 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12199 ins_cost(1.9 * INSN_COST);
12200 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12201
12202 ins_encode %{
12203 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12204 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12205 %}
12206 ins_pipe(ialu_reg_reg_shift);
12207 %}
12208
12209 // This pattern is automatically generated from aarch64_ad.m4
12210 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12211 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12212 %{
12213 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12214 ins_cost(1.9 * INSN_COST);
12215 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12216
12217 ins_encode %{
12218 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12219 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12220 %}
12221 ins_pipe(ialu_reg_reg_shift);
12222 %}
12223
12224 // This pattern is automatically generated from aarch64_ad.m4
12225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12226 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12227 %{
12228 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12229 ins_cost(1.9 * INSN_COST);
12230 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12231
12232 ins_encode %{
12233 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12234 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12235 %}
12236 ins_pipe(ialu_reg_reg_shift);
12237 %}
12238
12239 // This pattern is automatically generated from aarch64_ad.m4
12240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12241 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12242 %{
12243 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12244 ins_cost(1.9 * INSN_COST);
12245 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12246
12247 ins_encode %{
12248 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12249 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12250 %}
12251 ins_pipe(ialu_reg_reg_shift);
12252 %}
12253
12254 // This pattern is automatically generated from aarch64_ad.m4
12255 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12256 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12257 %{
12258 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12259 ins_cost(1.9 * INSN_COST);
12260 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12261
12262 ins_encode %{
12263 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12264 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12265 %}
12266 ins_pipe(ialu_reg_reg_shift);
12267 %}
12268
12269 // This pattern is automatically generated from aarch64_ad.m4
12270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12271 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12272 %{
12273 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12274 ins_cost(1.9 * INSN_COST);
12275 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12276
12277 ins_encode %{
12278 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12279 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12280 %}
12281 ins_pipe(ialu_reg_reg_shift);
12282 %}
12283
12284 // This pattern is automatically generated from aarch64_ad.m4
12285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12286 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12287 %{
12288 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12289 ins_cost(1.9 * INSN_COST);
12290 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12291
12292 ins_encode %{
12293 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12294 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12295 %}
12296 ins_pipe(ialu_reg_reg_shift);
12297 %}
12298
12299 // This pattern is automatically generated from aarch64_ad.m4
12300 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12301 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12302 %{
12303 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12304 ins_cost(1.9 * INSN_COST);
12305 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12306
12307 ins_encode %{
12308 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12309 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12310 %}
12311 ins_pipe(ialu_reg_reg_shift);
12312 %}
12313
12314 // This pattern is automatically generated from aarch64_ad.m4
12315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12316 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12317 %{
12318 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12319 ins_cost(1.9 * INSN_COST);
12320 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12321
12322 ins_encode %{
12323 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12324 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12325 %}
12326 ins_pipe(ialu_reg_reg_shift);
12327 %}
12328
12329 // This pattern is automatically generated from aarch64_ad.m4
12330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12331 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12332 %{
12333 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12334 ins_cost(1.9 * INSN_COST);
12335 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12336
12337 ins_encode %{
12338 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12339 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12340 %}
12341 ins_pipe(ialu_reg_reg_shift);
12342 %}
12343
12344 // This pattern is automatically generated from aarch64_ad.m4
12345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12346 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12347 %{
12348 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12349 ins_cost(1.9 * INSN_COST);
12350 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12351
12352 ins_encode %{
12353 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12354 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12355 %}
12356 ins_pipe(ialu_reg_reg_shift);
12357 %}
12358
12359 // This pattern is automatically generated from aarch64_ad.m4
12360 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12361 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12362 %{
12363 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12364 ins_cost(1.9 * INSN_COST);
12365 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12366
12367 ins_encode %{
12368 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12369 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12370 %}
12371 ins_pipe(ialu_reg_reg_shift);
12372 %}
12373
12374 // This pattern is automatically generated from aarch64_ad.m4
12375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12376 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12377 %{
12378 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12379 ins_cost(1.9 * INSN_COST);
12380 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12381
12382 ins_encode %{
12383 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12384 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12385 %}
12386 ins_pipe(ialu_reg_reg_shift);
12387 %}
12388
12389 // This pattern is automatically generated from aarch64_ad.m4
12390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12391 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12392 %{
12393 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12394 ins_cost(1.9 * INSN_COST);
12395 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12396
12397 ins_encode %{
12398 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12399 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12400 %}
12401 ins_pipe(ialu_reg_reg_shift);
12402 %}
12403
12404 // This pattern is automatically generated from aarch64_ad.m4
12405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12406 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12407 %{
12408 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12409 ins_cost(1.9 * INSN_COST);
12410 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12411
12412 ins_encode %{
12413 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12414 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12415 %}
12416 ins_pipe(ialu_reg_reg_shift);
12417 %}
12418
12419 // This pattern is automatically generated from aarch64_ad.m4
12420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12421 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12422 %{
12423 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12424 ins_cost(1.9 * INSN_COST);
12425 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12426
12427 ins_encode %{
12428 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12429 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12430 %}
12431 ins_pipe(ialu_reg_reg_shift);
12432 %}
12433
12434 // This pattern is automatically generated from aarch64_ad.m4
12435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12436 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12437 %{
12438 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12439 ins_cost(1.9 * INSN_COST);
12440 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12441
12442 ins_encode %{
12443 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12444 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12445 %}
12446 ins_pipe(ialu_reg_reg_shift);
12447 %}
12448
12449 // This pattern is automatically generated from aarch64_ad.m4
12450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12451 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12452 %{
12453 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12454 ins_cost(1.9 * INSN_COST);
12455 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12456
12457 ins_encode %{
12458 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12459 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12460 %}
12461 ins_pipe(ialu_reg_reg_shift);
12462 %}
12463
12464 // This pattern is automatically generated from aarch64_ad.m4
12465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12466 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12467 %{
12468 effect(DEF dst, USE src1, USE src2, USE cr);
12469 ins_cost(INSN_COST * 2);
12470 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12471
12472 ins_encode %{
12473 __ cselw($dst$$Register,
12474 $src1$$Register,
12475 $src2$$Register,
12476 Assembler::LT);
12477 %}
12478 ins_pipe(icond_reg_reg);
12479 %}
12480
12481 // This pattern is automatically generated from aarch64_ad.m4
12482 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12483 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12484 %{
12485 effect(DEF dst, USE src1, USE src2, USE cr);
12486 ins_cost(INSN_COST * 2);
12487 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12488
12489 ins_encode %{
12490 __ cselw($dst$$Register,
12491 $src1$$Register,
12492 $src2$$Register,
12493 Assembler::GT);
12494 %}
12495 ins_pipe(icond_reg_reg);
12496 %}
12497
12498 // This pattern is automatically generated from aarch64_ad.m4
12499 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12500 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12501 %{
12502 effect(DEF dst, USE src1, USE cr);
12503 ins_cost(INSN_COST * 2);
12504 format %{ "cselw $dst, $src1, zr lt\t" %}
12505
12506 ins_encode %{
12507 __ cselw($dst$$Register,
12508 $src1$$Register,
12509 zr,
12510 Assembler::LT);
12511 %}
12512 ins_pipe(icond_reg);
12513 %}
12514
12515 // This pattern is automatically generated from aarch64_ad.m4
12516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12517 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12518 %{
12519 effect(DEF dst, USE src1, USE cr);
12520 ins_cost(INSN_COST * 2);
12521 format %{ "cselw $dst, $src1, zr gt\t" %}
12522
12523 ins_encode %{
12524 __ cselw($dst$$Register,
12525 $src1$$Register,
12526 zr,
12527 Assembler::GT);
12528 %}
12529 ins_pipe(icond_reg);
12530 %}
12531
12532 // This pattern is automatically generated from aarch64_ad.m4
12533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12534 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12535 %{
12536 effect(DEF dst, USE src1, USE cr);
12537 ins_cost(INSN_COST * 2);
12538 format %{ "csincw $dst, $src1, zr le\t" %}
12539
12540 ins_encode %{
12541 __ csincw($dst$$Register,
12542 $src1$$Register,
12543 zr,
12544 Assembler::LE);
12545 %}
12546 ins_pipe(icond_reg);
12547 %}
12548
12549 // This pattern is automatically generated from aarch64_ad.m4
12550 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12551 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12552 %{
12553 effect(DEF dst, USE src1, USE cr);
12554 ins_cost(INSN_COST * 2);
12555 format %{ "csincw $dst, $src1, zr gt\t" %}
12556
12557 ins_encode %{
12558 __ csincw($dst$$Register,
12559 $src1$$Register,
12560 zr,
12561 Assembler::GT);
12562 %}
12563 ins_pipe(icond_reg);
12564 %}
12565
12566 // This pattern is automatically generated from aarch64_ad.m4
12567 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12568 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12569 %{
12570 effect(DEF dst, USE src1, USE cr);
12571 ins_cost(INSN_COST * 2);
12572 format %{ "csinvw $dst, $src1, zr lt\t" %}
12573
12574 ins_encode %{
12575 __ csinvw($dst$$Register,
12576 $src1$$Register,
12577 zr,
12578 Assembler::LT);
12579 %}
12580 ins_pipe(icond_reg);
12581 %}
12582
12583 // This pattern is automatically generated from aarch64_ad.m4
12584 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12585 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12586 %{
12587 effect(DEF dst, USE src1, USE cr);
12588 ins_cost(INSN_COST * 2);
12589 format %{ "csinvw $dst, $src1, zr ge\t" %}
12590
12591 ins_encode %{
12592 __ csinvw($dst$$Register,
12593 $src1$$Register,
12594 zr,
12595 Assembler::GE);
12596 %}
12597 ins_pipe(icond_reg);
12598 %}
12599
12600 // This pattern is automatically generated from aarch64_ad.m4
12601 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12602 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12603 %{
12604 match(Set dst (MinI src imm));
12605 ins_cost(INSN_COST * 3);
12606 expand %{
12607 rFlagsReg cr;
12608 compI_reg_imm0(cr, src);
12609 cmovI_reg_imm0_lt(dst, src, cr);
12610 %}
12611 %}
12612
12613 // This pattern is automatically generated from aarch64_ad.m4
12614 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12615 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12616 %{
12617 match(Set dst (MinI imm src));
12618 ins_cost(INSN_COST * 3);
12619 expand %{
12620 rFlagsReg cr;
12621 compI_reg_imm0(cr, src);
12622 cmovI_reg_imm0_lt(dst, src, cr);
12623 %}
12624 %}
12625
12626 // This pattern is automatically generated from aarch64_ad.m4
12627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12628 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12629 %{
12630 match(Set dst (MinI src imm));
12631 ins_cost(INSN_COST * 3);
12632 expand %{
12633 rFlagsReg cr;
12634 compI_reg_imm0(cr, src);
12635 cmovI_reg_imm1_le(dst, src, cr);
12636 %}
12637 %}
12638
12639 // This pattern is automatically generated from aarch64_ad.m4
12640 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12641 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12642 %{
12643 match(Set dst (MinI imm src));
12644 ins_cost(INSN_COST * 3);
12645 expand %{
12646 rFlagsReg cr;
12647 compI_reg_imm0(cr, src);
12648 cmovI_reg_imm1_le(dst, src, cr);
12649 %}
12650 %}
12651
12652 // This pattern is automatically generated from aarch64_ad.m4
12653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12654 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12655 %{
12656 match(Set dst (MinI src imm));
12657 ins_cost(INSN_COST * 3);
12658 expand %{
12659 rFlagsReg cr;
12660 compI_reg_imm0(cr, src);
12661 cmovI_reg_immM1_lt(dst, src, cr);
12662 %}
12663 %}
12664
12665 // This pattern is automatically generated from aarch64_ad.m4
12666 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12667 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12668 %{
12669 match(Set dst (MinI imm src));
12670 ins_cost(INSN_COST * 3);
12671 expand %{
12672 rFlagsReg cr;
12673 compI_reg_imm0(cr, src);
12674 cmovI_reg_immM1_lt(dst, src, cr);
12675 %}
12676 %}
12677
12678 // This pattern is automatically generated from aarch64_ad.m4
12679 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12680 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12681 %{
12682 match(Set dst (MaxI src imm));
12683 ins_cost(INSN_COST * 3);
12684 expand %{
12685 rFlagsReg cr;
12686 compI_reg_imm0(cr, src);
12687 cmovI_reg_imm0_gt(dst, src, cr);
12688 %}
12689 %}
12690
12691 // This pattern is automatically generated from aarch64_ad.m4
12692 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12693 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12694 %{
12695 match(Set dst (MaxI imm src));
12696 ins_cost(INSN_COST * 3);
12697 expand %{
12698 rFlagsReg cr;
12699 compI_reg_imm0(cr, src);
12700 cmovI_reg_imm0_gt(dst, src, cr);
12701 %}
12702 %}
12703
12704 // This pattern is automatically generated from aarch64_ad.m4
12705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12706 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12707 %{
12708 match(Set dst (MaxI src imm));
12709 ins_cost(INSN_COST * 3);
12710 expand %{
12711 rFlagsReg cr;
12712 compI_reg_imm0(cr, src);
12713 cmovI_reg_imm1_gt(dst, src, cr);
12714 %}
12715 %}
12716
12717 // This pattern is automatically generated from aarch64_ad.m4
12718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12719 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12720 %{
12721 match(Set dst (MaxI imm src));
12722 ins_cost(INSN_COST * 3);
12723 expand %{
12724 rFlagsReg cr;
12725 compI_reg_imm0(cr, src);
12726 cmovI_reg_imm1_gt(dst, src, cr);
12727 %}
12728 %}
12729
12730 // This pattern is automatically generated from aarch64_ad.m4
12731 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12732 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12733 %{
12734 match(Set dst (MaxI src imm));
12735 ins_cost(INSN_COST * 3);
12736 expand %{
12737 rFlagsReg cr;
12738 compI_reg_imm0(cr, src);
12739 cmovI_reg_immM1_ge(dst, src, cr);
12740 %}
12741 %}
12742
12743 // This pattern is automatically generated from aarch64_ad.m4
12744 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12745 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12746 %{
12747 match(Set dst (MaxI imm src));
12748 ins_cost(INSN_COST * 3);
12749 expand %{
12750 rFlagsReg cr;
12751 compI_reg_imm0(cr, src);
12752 cmovI_reg_immM1_ge(dst, src, cr);
12753 %}
12754 %}
12755
12756 // This pattern is automatically generated from aarch64_ad.m4
12757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12758 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12759 %{
12760 match(Set dst (ReverseI src));
12761 ins_cost(INSN_COST);
12762 format %{ "rbitw $dst, $src" %}
12763 ins_encode %{
12764 __ rbitw($dst$$Register, $src$$Register);
12765 %}
12766 ins_pipe(ialu_reg);
12767 %}
12768
12769 // This pattern is automatically generated from aarch64_ad.m4
12770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12771 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12772 %{
12773 match(Set dst (ReverseL src));
12774 ins_cost(INSN_COST);
12775 format %{ "rbit $dst, $src" %}
12776 ins_encode %{
12777 __ rbit($dst$$Register, $src$$Register);
12778 %}
12779 ins_pipe(ialu_reg);
12780 %}
12781
12782
12783 // END This section of the file is automatically generated. Do not edit --------------
12784
12785
12786 // ============================================================================
12787 // Floating Point Arithmetic Instructions
12788
12789 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12790 match(Set dst (AddHF src1 src2));
12791 format %{ "faddh $dst, $src1, $src2" %}
12792 ins_encode %{
12793 __ faddh($dst$$FloatRegister,
12794 $src1$$FloatRegister,
12795 $src2$$FloatRegister);
12796 %}
12797 ins_pipe(fp_dop_reg_reg_s);
12798 %}
12799
12800 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12801 match(Set dst (AddF src1 src2));
12802
12803 ins_cost(INSN_COST * 5);
12804 format %{ "fadds $dst, $src1, $src2" %}
12805
12806 ins_encode %{
12807 __ fadds(as_FloatRegister($dst$$reg),
12808 as_FloatRegister($src1$$reg),
12809 as_FloatRegister($src2$$reg));
12810 %}
12811
12812 ins_pipe(fp_dop_reg_reg_s);
12813 %}
12814
12815 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12816 match(Set dst (AddD src1 src2));
12817
12818 ins_cost(INSN_COST * 5);
12819 format %{ "faddd $dst, $src1, $src2" %}
12820
12821 ins_encode %{
12822 __ faddd(as_FloatRegister($dst$$reg),
12823 as_FloatRegister($src1$$reg),
12824 as_FloatRegister($src2$$reg));
12825 %}
12826
12827 ins_pipe(fp_dop_reg_reg_d);
12828 %}
12829
12830 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12831 match(Set dst (SubHF src1 src2));
12832 format %{ "fsubh $dst, $src1, $src2" %}
12833 ins_encode %{
12834 __ fsubh($dst$$FloatRegister,
12835 $src1$$FloatRegister,
12836 $src2$$FloatRegister);
12837 %}
12838 ins_pipe(fp_dop_reg_reg_s);
12839 %}
12840
12841 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12842 match(Set dst (SubF src1 src2));
12843
12844 ins_cost(INSN_COST * 5);
12845 format %{ "fsubs $dst, $src1, $src2" %}
12846
12847 ins_encode %{
12848 __ fsubs(as_FloatRegister($dst$$reg),
12849 as_FloatRegister($src1$$reg),
12850 as_FloatRegister($src2$$reg));
12851 %}
12852
12853 ins_pipe(fp_dop_reg_reg_s);
12854 %}
12855
12856 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12857 match(Set dst (SubD src1 src2));
12858
12859 ins_cost(INSN_COST * 5);
12860 format %{ "fsubd $dst, $src1, $src2" %}
12861
12862 ins_encode %{
12863 __ fsubd(as_FloatRegister($dst$$reg),
12864 as_FloatRegister($src1$$reg),
12865 as_FloatRegister($src2$$reg));
12866 %}
12867
12868 ins_pipe(fp_dop_reg_reg_d);
12869 %}
12870
12871 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12872 match(Set dst (MulHF src1 src2));
12873 format %{ "fmulh $dst, $src1, $src2" %}
12874 ins_encode %{
12875 __ fmulh($dst$$FloatRegister,
12876 $src1$$FloatRegister,
12877 $src2$$FloatRegister);
12878 %}
12879 ins_pipe(fp_dop_reg_reg_s);
12880 %}
12881
12882 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12883 match(Set dst (MulF src1 src2));
12884
12885 ins_cost(INSN_COST * 6);
12886 format %{ "fmuls $dst, $src1, $src2" %}
12887
12888 ins_encode %{
12889 __ fmuls(as_FloatRegister($dst$$reg),
12890 as_FloatRegister($src1$$reg),
12891 as_FloatRegister($src2$$reg));
12892 %}
12893
12894 ins_pipe(fp_dop_reg_reg_s);
12895 %}
12896
12897 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12898 match(Set dst (MulD src1 src2));
12899
12900 ins_cost(INSN_COST * 6);
12901 format %{ "fmuld $dst, $src1, $src2" %}
12902
12903 ins_encode %{
12904 __ fmuld(as_FloatRegister($dst$$reg),
12905 as_FloatRegister($src1$$reg),
12906 as_FloatRegister($src2$$reg));
12907 %}
12908
12909 ins_pipe(fp_dop_reg_reg_d);
12910 %}
12911
12912 // src1 * src2 + src3 (half-precision float)
12913 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12914 match(Set dst (FmaHF src3 (Binary src1 src2)));
12915 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12916 ins_encode %{
12917 assert(UseFMA, "Needs FMA instructions support.");
12918 __ fmaddh($dst$$FloatRegister,
12919 $src1$$FloatRegister,
12920 $src2$$FloatRegister,
12921 $src3$$FloatRegister);
12922 %}
12923 ins_pipe(pipe_class_default);
12924 %}
12925
12926 // src1 * src2 + src3
12927 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12928 match(Set dst (FmaF src3 (Binary src1 src2)));
12929
12930 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12931
12932 ins_encode %{
12933 assert(UseFMA, "Needs FMA instructions support.");
12934 __ fmadds(as_FloatRegister($dst$$reg),
12935 as_FloatRegister($src1$$reg),
12936 as_FloatRegister($src2$$reg),
12937 as_FloatRegister($src3$$reg));
12938 %}
12939
12940 ins_pipe(pipe_class_default);
12941 %}
12942
12943 // src1 * src2 + src3
12944 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12945 match(Set dst (FmaD src3 (Binary src1 src2)));
12946
12947 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12948
12949 ins_encode %{
12950 assert(UseFMA, "Needs FMA instructions support.");
12951 __ fmaddd(as_FloatRegister($dst$$reg),
12952 as_FloatRegister($src1$$reg),
12953 as_FloatRegister($src2$$reg),
12954 as_FloatRegister($src3$$reg));
12955 %}
12956
12957 ins_pipe(pipe_class_default);
12958 %}
12959
12960 // src1 * (-src2) + src3
12961 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12962 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12963 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12964
12965 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
12966
12967 ins_encode %{
12968 assert(UseFMA, "Needs FMA instructions support.");
12969 __ fmsubs(as_FloatRegister($dst$$reg),
12970 as_FloatRegister($src1$$reg),
12971 as_FloatRegister($src2$$reg),
12972 as_FloatRegister($src3$$reg));
12973 %}
12974
12975 ins_pipe(pipe_class_default);
12976 %}
12977
12978 // src1 * (-src2) + src3
12979 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12980 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12981 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12982
12983 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
12984
12985 ins_encode %{
12986 assert(UseFMA, "Needs FMA instructions support.");
12987 __ fmsubd(as_FloatRegister($dst$$reg),
12988 as_FloatRegister($src1$$reg),
12989 as_FloatRegister($src2$$reg),
12990 as_FloatRegister($src3$$reg));
12991 %}
12992
12993 ins_pipe(pipe_class_default);
12994 %}
12995
12996 // src1 * (-src2) - src3
12997 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
12998 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12999 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13000
13001 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13002
13003 ins_encode %{
13004 assert(UseFMA, "Needs FMA instructions support.");
13005 __ fnmadds(as_FloatRegister($dst$$reg),
13006 as_FloatRegister($src1$$reg),
13007 as_FloatRegister($src2$$reg),
13008 as_FloatRegister($src3$$reg));
13009 %}
13010
13011 ins_pipe(pipe_class_default);
13012 %}
13013
13014 // src1 * (-src2) - src3
13015 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13016 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13017 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13018
13019 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13020
13021 ins_encode %{
13022 assert(UseFMA, "Needs FMA instructions support.");
13023 __ fnmaddd(as_FloatRegister($dst$$reg),
13024 as_FloatRegister($src1$$reg),
13025 as_FloatRegister($src2$$reg),
13026 as_FloatRegister($src3$$reg));
13027 %}
13028
13029 ins_pipe(pipe_class_default);
13030 %}
13031
13032 // src1 * src2 - src3
13033 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13034 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13035
13036 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13037
13038 ins_encode %{
13039 assert(UseFMA, "Needs FMA instructions support.");
13040 __ fnmsubs(as_FloatRegister($dst$$reg),
13041 as_FloatRegister($src1$$reg),
13042 as_FloatRegister($src2$$reg),
13043 as_FloatRegister($src3$$reg));
13044 %}
13045
13046 ins_pipe(pipe_class_default);
13047 %}
13048
13049 // src1 * src2 - src3
13050 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13051 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13052
13053 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13054
13055 ins_encode %{
13056 assert(UseFMA, "Needs FMA instructions support.");
13057 // n.b. insn name should be fnmsubd
13058 __ fnmsub(as_FloatRegister($dst$$reg),
13059 as_FloatRegister($src1$$reg),
13060 as_FloatRegister($src2$$reg),
13061 as_FloatRegister($src3$$reg));
13062 %}
13063
13064 ins_pipe(pipe_class_default);
13065 %}
13066
13067 // Math.max(HH)H (half-precision float)
13068 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13069 match(Set dst (MaxHF src1 src2));
13070 format %{ "fmaxh $dst, $src1, $src2" %}
13071 ins_encode %{
13072 __ fmaxh($dst$$FloatRegister,
13073 $src1$$FloatRegister,
13074 $src2$$FloatRegister);
13075 %}
13076 ins_pipe(fp_dop_reg_reg_s);
13077 %}
13078
13079 // Math.min(HH)H (half-precision float)
13080 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13081 match(Set dst (MinHF src1 src2));
13082 format %{ "fminh $dst, $src1, $src2" %}
13083 ins_encode %{
13084 __ fminh($dst$$FloatRegister,
13085 $src1$$FloatRegister,
13086 $src2$$FloatRegister);
13087 %}
13088 ins_pipe(fp_dop_reg_reg_s);
13089 %}
13090
13091 // Math.max(FF)F
13092 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13093 match(Set dst (MaxF src1 src2));
13094
13095 format %{ "fmaxs $dst, $src1, $src2" %}
13096 ins_encode %{
13097 __ fmaxs(as_FloatRegister($dst$$reg),
13098 as_FloatRegister($src1$$reg),
13099 as_FloatRegister($src2$$reg));
13100 %}
13101
13102 ins_pipe(fp_dop_reg_reg_s);
13103 %}
13104
13105 // Math.min(FF)F
13106 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13107 match(Set dst (MinF src1 src2));
13108
13109 format %{ "fmins $dst, $src1, $src2" %}
13110 ins_encode %{
13111 __ fmins(as_FloatRegister($dst$$reg),
13112 as_FloatRegister($src1$$reg),
13113 as_FloatRegister($src2$$reg));
13114 %}
13115
13116 ins_pipe(fp_dop_reg_reg_s);
13117 %}
13118
13119 // Math.max(DD)D
13120 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13121 match(Set dst (MaxD src1 src2));
13122
13123 format %{ "fmaxd $dst, $src1, $src2" %}
13124 ins_encode %{
13125 __ fmaxd(as_FloatRegister($dst$$reg),
13126 as_FloatRegister($src1$$reg),
13127 as_FloatRegister($src2$$reg));
13128 %}
13129
13130 ins_pipe(fp_dop_reg_reg_d);
13131 %}
13132
13133 // Math.min(DD)D
13134 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13135 match(Set dst (MinD src1 src2));
13136
13137 format %{ "fmind $dst, $src1, $src2" %}
13138 ins_encode %{
13139 __ fmind(as_FloatRegister($dst$$reg),
13140 as_FloatRegister($src1$$reg),
13141 as_FloatRegister($src2$$reg));
13142 %}
13143
13144 ins_pipe(fp_dop_reg_reg_d);
13145 %}
13146
13147 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13148 match(Set dst (DivHF src1 src2));
13149 format %{ "fdivh $dst, $src1, $src2" %}
13150 ins_encode %{
13151 __ fdivh($dst$$FloatRegister,
13152 $src1$$FloatRegister,
13153 $src2$$FloatRegister);
13154 %}
13155 ins_pipe(fp_div_s);
13156 %}
13157
13158 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13159 match(Set dst (DivF src1 src2));
13160
13161 ins_cost(INSN_COST * 18);
13162 format %{ "fdivs $dst, $src1, $src2" %}
13163
13164 ins_encode %{
13165 __ fdivs(as_FloatRegister($dst$$reg),
13166 as_FloatRegister($src1$$reg),
13167 as_FloatRegister($src2$$reg));
13168 %}
13169
13170 ins_pipe(fp_div_s);
13171 %}
13172
13173 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13174 match(Set dst (DivD src1 src2));
13175
13176 ins_cost(INSN_COST * 32);
13177 format %{ "fdivd $dst, $src1, $src2" %}
13178
13179 ins_encode %{
13180 __ fdivd(as_FloatRegister($dst$$reg),
13181 as_FloatRegister($src1$$reg),
13182 as_FloatRegister($src2$$reg));
13183 %}
13184
13185 ins_pipe(fp_div_d);
13186 %}
13187
13188 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13189 match(Set dst (NegF src));
13190
13191 ins_cost(INSN_COST * 3);
13192 format %{ "fneg $dst, $src" %}
13193
13194 ins_encode %{
13195 __ fnegs(as_FloatRegister($dst$$reg),
13196 as_FloatRegister($src$$reg));
13197 %}
13198
13199 ins_pipe(fp_uop_s);
13200 %}
13201
13202 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13203 match(Set dst (NegD src));
13204
13205 ins_cost(INSN_COST * 3);
13206 format %{ "fnegd $dst, $src" %}
13207
13208 ins_encode %{
13209 __ fnegd(as_FloatRegister($dst$$reg),
13210 as_FloatRegister($src$$reg));
13211 %}
13212
13213 ins_pipe(fp_uop_d);
13214 %}
13215
13216 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13217 %{
13218 match(Set dst (AbsI src));
13219
13220 effect(KILL cr);
13221 ins_cost(INSN_COST * 2);
13222 format %{ "cmpw $src, zr\n\t"
13223 "cnegw $dst, $src, Assembler::LT\t# int abs"
13224 %}
13225
13226 ins_encode %{
13227 __ cmpw(as_Register($src$$reg), zr);
13228 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13229 %}
13230 ins_pipe(pipe_class_default);
13231 %}
13232
13233 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13234 %{
13235 match(Set dst (AbsL src));
13236
13237 effect(KILL cr);
13238 ins_cost(INSN_COST * 2);
13239 format %{ "cmp $src, zr\n\t"
13240 "cneg $dst, $src, Assembler::LT\t# long abs"
13241 %}
13242
13243 ins_encode %{
13244 __ cmp(as_Register($src$$reg), zr);
13245 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13246 %}
13247 ins_pipe(pipe_class_default);
13248 %}
13249
13250 instruct absF_reg(vRegF dst, vRegF src) %{
13251 match(Set dst (AbsF src));
13252
13253 ins_cost(INSN_COST * 3);
13254 format %{ "fabss $dst, $src" %}
13255 ins_encode %{
13256 __ fabss(as_FloatRegister($dst$$reg),
13257 as_FloatRegister($src$$reg));
13258 %}
13259
13260 ins_pipe(fp_uop_s);
13261 %}
13262
13263 instruct absD_reg(vRegD dst, vRegD src) %{
13264 match(Set dst (AbsD src));
13265
13266 ins_cost(INSN_COST * 3);
13267 format %{ "fabsd $dst, $src" %}
13268 ins_encode %{
13269 __ fabsd(as_FloatRegister($dst$$reg),
13270 as_FloatRegister($src$$reg));
13271 %}
13272
13273 ins_pipe(fp_uop_d);
13274 %}
13275
13276 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13277 match(Set dst (AbsF (SubF src1 src2)));
13278
13279 ins_cost(INSN_COST * 3);
13280 format %{ "fabds $dst, $src1, $src2" %}
13281 ins_encode %{
13282 __ fabds(as_FloatRegister($dst$$reg),
13283 as_FloatRegister($src1$$reg),
13284 as_FloatRegister($src2$$reg));
13285 %}
13286
13287 ins_pipe(fp_uop_s);
13288 %}
13289
13290 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13291 match(Set dst (AbsD (SubD src1 src2)));
13292
13293 ins_cost(INSN_COST * 3);
13294 format %{ "fabdd $dst, $src1, $src2" %}
13295 ins_encode %{
13296 __ fabdd(as_FloatRegister($dst$$reg),
13297 as_FloatRegister($src1$$reg),
13298 as_FloatRegister($src2$$reg));
13299 %}
13300
13301 ins_pipe(fp_uop_d);
13302 %}
13303
13304 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13305 match(Set dst (SqrtD src));
13306
13307 ins_cost(INSN_COST * 50);
13308 format %{ "fsqrtd $dst, $src" %}
13309 ins_encode %{
13310 __ fsqrtd(as_FloatRegister($dst$$reg),
13311 as_FloatRegister($src$$reg));
13312 %}
13313
13314 ins_pipe(fp_div_s);
13315 %}
13316
13317 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13318 match(Set dst (SqrtF src));
13319
13320 ins_cost(INSN_COST * 50);
13321 format %{ "fsqrts $dst, $src" %}
13322 ins_encode %{
13323 __ fsqrts(as_FloatRegister($dst$$reg),
13324 as_FloatRegister($src$$reg));
13325 %}
13326
13327 ins_pipe(fp_div_d);
13328 %}
13329
13330 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13331 match(Set dst (SqrtHF src));
13332 format %{ "fsqrth $dst, $src" %}
13333 ins_encode %{
13334 __ fsqrth($dst$$FloatRegister,
13335 $src$$FloatRegister);
13336 %}
13337 ins_pipe(fp_div_s);
13338 %}
13339
13340 // Math.rint, floor, ceil
13341 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13342 match(Set dst (RoundDoubleMode src rmode));
13343 format %{ "frint $dst, $src, $rmode" %}
13344 ins_encode %{
13345 switch ($rmode$$constant) {
13346 case RoundDoubleModeNode::rmode_rint:
13347 __ frintnd(as_FloatRegister($dst$$reg),
13348 as_FloatRegister($src$$reg));
13349 break;
13350 case RoundDoubleModeNode::rmode_floor:
13351 __ frintmd(as_FloatRegister($dst$$reg),
13352 as_FloatRegister($src$$reg));
13353 break;
13354 case RoundDoubleModeNode::rmode_ceil:
13355 __ frintpd(as_FloatRegister($dst$$reg),
13356 as_FloatRegister($src$$reg));
13357 break;
13358 }
13359 %}
13360 ins_pipe(fp_uop_d);
13361 %}
13362
13363 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13364 match(Set dst (CopySignD src1 (Binary src2 zero)));
13365 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13366 format %{ "CopySignD $dst $src1 $src2" %}
13367 ins_encode %{
13368 FloatRegister dst = as_FloatRegister($dst$$reg),
13369 src1 = as_FloatRegister($src1$$reg),
13370 src2 = as_FloatRegister($src2$$reg),
13371 zero = as_FloatRegister($zero$$reg);
13372 __ fnegd(dst, zero);
13373 __ bsl(dst, __ T8B, src2, src1);
13374 %}
13375 ins_pipe(fp_uop_d);
13376 %}
13377
13378 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13379 match(Set dst (CopySignF src1 src2));
13380 effect(TEMP_DEF dst, USE src1, USE src2);
13381 format %{ "CopySignF $dst $src1 $src2" %}
13382 ins_encode %{
13383 FloatRegister dst = as_FloatRegister($dst$$reg),
13384 src1 = as_FloatRegister($src1$$reg),
13385 src2 = as_FloatRegister($src2$$reg);
13386 __ movi(dst, __ T2S, 0x80, 24);
13387 __ bsl(dst, __ T8B, src2, src1);
13388 %}
13389 ins_pipe(fp_uop_d);
13390 %}
13391
13392 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13393 match(Set dst (SignumD src (Binary zero one)));
13394 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13395 format %{ "signumD $dst, $src" %}
13396 ins_encode %{
13397 FloatRegister src = as_FloatRegister($src$$reg),
13398 dst = as_FloatRegister($dst$$reg),
13399 zero = as_FloatRegister($zero$$reg),
13400 one = as_FloatRegister($one$$reg);
13401 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13402 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13403 // Bit selection instruction gets bit from "one" for each enabled bit in
13404 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13405 // NaN the whole "src" will be copied because "dst" is zero. For all other
13406 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13407 // from "src", and all other bits are copied from 1.0.
13408 __ bsl(dst, __ T8B, one, src);
13409 %}
13410 ins_pipe(fp_uop_d);
13411 %}
13412
13413 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13414 match(Set dst (SignumF src (Binary zero one)));
13415 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13416 format %{ "signumF $dst, $src" %}
13417 ins_encode %{
13418 FloatRegister src = as_FloatRegister($src$$reg),
13419 dst = as_FloatRegister($dst$$reg),
13420 zero = as_FloatRegister($zero$$reg),
13421 one = as_FloatRegister($one$$reg);
13422 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13423 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13424 // Bit selection instruction gets bit from "one" for each enabled bit in
13425 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13426 // NaN the whole "src" will be copied because "dst" is zero. For all other
13427 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13428 // from "src", and all other bits are copied from 1.0.
13429 __ bsl(dst, __ T8B, one, src);
13430 %}
13431 ins_pipe(fp_uop_d);
13432 %}
13433
13434 instruct onspinwait() %{
13435 match(OnSpinWait);
13436 ins_cost(INSN_COST);
13437
13438 format %{ "onspinwait" %}
13439
13440 ins_encode %{
13441 __ spin_wait();
13442 %}
13443 ins_pipe(pipe_class_empty);
13444 %}
13445
13446 // ============================================================================
13447 // Logical Instructions
13448
13449 // Integer Logical Instructions
13450
13451 // And Instructions
13452
13453
13454 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13455 match(Set dst (AndI src1 src2));
13456
13457 format %{ "andw $dst, $src1, $src2\t# int" %}
13458
13459 ins_cost(INSN_COST);
13460 ins_encode %{
13461 __ andw(as_Register($dst$$reg),
13462 as_Register($src1$$reg),
13463 as_Register($src2$$reg));
13464 %}
13465
13466 ins_pipe(ialu_reg_reg);
13467 %}
13468
13469 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13470 match(Set dst (AndI src1 src2));
13471
13472 format %{ "andsw $dst, $src1, $src2\t# int" %}
13473
13474 ins_cost(INSN_COST);
13475 ins_encode %{
13476 __ andw(as_Register($dst$$reg),
13477 as_Register($src1$$reg),
13478 (uint64_t)($src2$$constant));
13479 %}
13480
13481 ins_pipe(ialu_reg_imm);
13482 %}
13483
13484 // Or Instructions
13485
13486 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13487 match(Set dst (OrI src1 src2));
13488
13489 format %{ "orrw $dst, $src1, $src2\t# int" %}
13490
13491 ins_cost(INSN_COST);
13492 ins_encode %{
13493 __ orrw(as_Register($dst$$reg),
13494 as_Register($src1$$reg),
13495 as_Register($src2$$reg));
13496 %}
13497
13498 ins_pipe(ialu_reg_reg);
13499 %}
13500
13501 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13502 match(Set dst (OrI src1 src2));
13503
13504 format %{ "orrw $dst, $src1, $src2\t# int" %}
13505
13506 ins_cost(INSN_COST);
13507 ins_encode %{
13508 __ orrw(as_Register($dst$$reg),
13509 as_Register($src1$$reg),
13510 (uint64_t)($src2$$constant));
13511 %}
13512
13513 ins_pipe(ialu_reg_imm);
13514 %}
13515
13516 // Xor Instructions
13517
13518 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13519 match(Set dst (XorI src1 src2));
13520
13521 format %{ "eorw $dst, $src1, $src2\t# int" %}
13522
13523 ins_cost(INSN_COST);
13524 ins_encode %{
13525 __ eorw(as_Register($dst$$reg),
13526 as_Register($src1$$reg),
13527 as_Register($src2$$reg));
13528 %}
13529
13530 ins_pipe(ialu_reg_reg);
13531 %}
13532
13533 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13534 match(Set dst (XorI src1 src2));
13535
13536 format %{ "eorw $dst, $src1, $src2\t# int" %}
13537
13538 ins_cost(INSN_COST);
13539 ins_encode %{
13540 __ eorw(as_Register($dst$$reg),
13541 as_Register($src1$$reg),
13542 (uint64_t)($src2$$constant));
13543 %}
13544
13545 ins_pipe(ialu_reg_imm);
13546 %}
13547
13548 // Long Logical Instructions
13549 // TODO
13550
13551 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13552 match(Set dst (AndL src1 src2));
13553
13554 format %{ "and $dst, $src1, $src2\t# int" %}
13555
13556 ins_cost(INSN_COST);
13557 ins_encode %{
13558 __ andr(as_Register($dst$$reg),
13559 as_Register($src1$$reg),
13560 as_Register($src2$$reg));
13561 %}
13562
13563 ins_pipe(ialu_reg_reg);
13564 %}
13565
13566 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13567 match(Set dst (AndL src1 src2));
13568
13569 format %{ "and $dst, $src1, $src2\t# int" %}
13570
13571 ins_cost(INSN_COST);
13572 ins_encode %{
13573 __ andr(as_Register($dst$$reg),
13574 as_Register($src1$$reg),
13575 (uint64_t)($src2$$constant));
13576 %}
13577
13578 ins_pipe(ialu_reg_imm);
13579 %}
13580
13581 // Or Instructions
13582
13583 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13584 match(Set dst (OrL src1 src2));
13585
13586 format %{ "orr $dst, $src1, $src2\t# int" %}
13587
13588 ins_cost(INSN_COST);
13589 ins_encode %{
13590 __ orr(as_Register($dst$$reg),
13591 as_Register($src1$$reg),
13592 as_Register($src2$$reg));
13593 %}
13594
13595 ins_pipe(ialu_reg_reg);
13596 %}
13597
13598 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13599 match(Set dst (OrL src1 src2));
13600
13601 format %{ "orr $dst, $src1, $src2\t# int" %}
13602
13603 ins_cost(INSN_COST);
13604 ins_encode %{
13605 __ orr(as_Register($dst$$reg),
13606 as_Register($src1$$reg),
13607 (uint64_t)($src2$$constant));
13608 %}
13609
13610 ins_pipe(ialu_reg_imm);
13611 %}
13612
13613 // Xor Instructions
13614
13615 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13616 match(Set dst (XorL src1 src2));
13617
13618 format %{ "eor $dst, $src1, $src2\t# int" %}
13619
13620 ins_cost(INSN_COST);
13621 ins_encode %{
13622 __ eor(as_Register($dst$$reg),
13623 as_Register($src1$$reg),
13624 as_Register($src2$$reg));
13625 %}
13626
13627 ins_pipe(ialu_reg_reg);
13628 %}
13629
13630 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13631 match(Set dst (XorL src1 src2));
13632
13633 ins_cost(INSN_COST);
13634 format %{ "eor $dst, $src1, $src2\t# int" %}
13635
13636 ins_encode %{
13637 __ eor(as_Register($dst$$reg),
13638 as_Register($src1$$reg),
13639 (uint64_t)($src2$$constant));
13640 %}
13641
13642 ins_pipe(ialu_reg_imm);
13643 %}
13644
13645 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13646 %{
13647 match(Set dst (ConvI2L src));
13648
13649 ins_cost(INSN_COST);
13650 format %{ "sxtw $dst, $src\t# i2l" %}
13651 ins_encode %{
13652 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13653 %}
13654 ins_pipe(ialu_reg_shift);
13655 %}
13656
13657 // this pattern occurs in bigmath arithmetic
13658 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13659 %{
13660 match(Set dst (AndL (ConvI2L src) mask));
13661
13662 ins_cost(INSN_COST);
13663 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13664 ins_encode %{
13665 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13666 %}
13667
13668 ins_pipe(ialu_reg_shift);
13669 %}
13670
13671 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13672 match(Set dst (ConvL2I src));
13673
13674 ins_cost(INSN_COST);
13675 format %{ "movw $dst, $src \t// l2i" %}
13676
13677 ins_encode %{
13678 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13679 %}
13680
13681 ins_pipe(ialu_reg);
13682 %}
13683
13684 instruct convD2F_reg(vRegF dst, vRegD src) %{
13685 match(Set dst (ConvD2F src));
13686
13687 ins_cost(INSN_COST * 5);
13688 format %{ "fcvtd $dst, $src \t// d2f" %}
13689
13690 ins_encode %{
13691 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13692 %}
13693
13694 ins_pipe(fp_d2f);
13695 %}
13696
13697 instruct convF2D_reg(vRegD dst, vRegF src) %{
13698 match(Set dst (ConvF2D src));
13699
13700 ins_cost(INSN_COST * 5);
13701 format %{ "fcvts $dst, $src \t// f2d" %}
13702
13703 ins_encode %{
13704 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13705 %}
13706
13707 ins_pipe(fp_f2d);
13708 %}
13709
13710 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13711 match(Set dst (ConvF2I src));
13712
13713 ins_cost(INSN_COST * 5);
13714 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13715
13716 ins_encode %{
13717 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13718 %}
13719
13720 ins_pipe(fp_f2i);
13721 %}
13722
13723 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13724 match(Set dst (ConvF2L src));
13725
13726 ins_cost(INSN_COST * 5);
13727 format %{ "fcvtzs $dst, $src \t// f2l" %}
13728
13729 ins_encode %{
13730 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13731 %}
13732
13733 ins_pipe(fp_f2l);
13734 %}
13735
13736 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13737 match(Set dst (ConvF2HF src));
13738 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13739 "smov $dst, $tmp\t# move result from $tmp to $dst"
13740 %}
13741 effect(TEMP tmp);
13742 ins_encode %{
13743 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13744 %}
13745 ins_pipe(pipe_slow);
13746 %}
13747
13748 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13749 match(Set dst (ConvHF2F src));
13750 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13751 "fcvt $dst, $tmp\t# convert half to single precision"
13752 %}
13753 effect(TEMP tmp);
13754 ins_encode %{
13755 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13756 %}
13757 ins_pipe(pipe_slow);
13758 %}
13759
13760 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13761 match(Set dst (ConvI2F src));
13762
13763 ins_cost(INSN_COST * 5);
13764 format %{ "scvtfws $dst, $src \t// i2f" %}
13765
13766 ins_encode %{
13767 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13768 %}
13769
13770 ins_pipe(fp_i2f);
13771 %}
13772
13773 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13774 match(Set dst (ConvL2F src));
13775
13776 ins_cost(INSN_COST * 5);
13777 format %{ "scvtfs $dst, $src \t// l2f" %}
13778
13779 ins_encode %{
13780 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13781 %}
13782
13783 ins_pipe(fp_l2f);
13784 %}
13785
13786 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13787 match(Set dst (ConvD2I src));
13788
13789 ins_cost(INSN_COST * 5);
13790 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13791
13792 ins_encode %{
13793 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13794 %}
13795
13796 ins_pipe(fp_d2i);
13797 %}
13798
13799 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13800 match(Set dst (ConvD2L src));
13801
13802 ins_cost(INSN_COST * 5);
13803 format %{ "fcvtzd $dst, $src \t// d2l" %}
13804
13805 ins_encode %{
13806 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13807 %}
13808
13809 ins_pipe(fp_d2l);
13810 %}
13811
13812 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13813 match(Set dst (ConvI2D src));
13814
13815 ins_cost(INSN_COST * 5);
13816 format %{ "scvtfwd $dst, $src \t// i2d" %}
13817
13818 ins_encode %{
13819 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13820 %}
13821
13822 ins_pipe(fp_i2d);
13823 %}
13824
13825 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13826 match(Set dst (ConvL2D src));
13827
13828 ins_cost(INSN_COST * 5);
13829 format %{ "scvtfd $dst, $src \t// l2d" %}
13830
13831 ins_encode %{
13832 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13833 %}
13834
13835 ins_pipe(fp_l2d);
13836 %}
13837
13838 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13839 %{
13840 match(Set dst (RoundD src));
13841 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13842 format %{ "java_round_double $dst,$src"%}
13843 ins_encode %{
13844 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13845 as_FloatRegister($ftmp$$reg));
13846 %}
13847 ins_pipe(pipe_slow);
13848 %}
13849
13850 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13851 %{
13852 match(Set dst (RoundF src));
13853 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13854 format %{ "java_round_float $dst,$src"%}
13855 ins_encode %{
13856 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13857 as_FloatRegister($ftmp$$reg));
13858 %}
13859 ins_pipe(pipe_slow);
13860 %}
13861
13862 // stack <-> reg and reg <-> reg shuffles with no conversion
13863
13864 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13865
13866 match(Set dst (MoveF2I src));
13867
13868 effect(DEF dst, USE src);
13869
13870 ins_cost(4 * INSN_COST);
13871
13872 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13873
13874 ins_encode %{
13875 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13876 %}
13877
13878 ins_pipe(iload_reg_reg);
13879
13880 %}
13881
13882 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13883
13884 match(Set dst (MoveI2F src));
13885
13886 effect(DEF dst, USE src);
13887
13888 ins_cost(4 * INSN_COST);
13889
13890 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13891
13892 ins_encode %{
13893 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13894 %}
13895
13896 ins_pipe(pipe_class_memory);
13897
13898 %}
13899
13900 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13901
13902 match(Set dst (MoveD2L src));
13903
13904 effect(DEF dst, USE src);
13905
13906 ins_cost(4 * INSN_COST);
13907
13908 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13909
13910 ins_encode %{
13911 __ ldr($dst$$Register, Address(sp, $src$$disp));
13912 %}
13913
13914 ins_pipe(iload_reg_reg);
13915
13916 %}
13917
13918 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13919
13920 match(Set dst (MoveL2D src));
13921
13922 effect(DEF dst, USE src);
13923
13924 ins_cost(4 * INSN_COST);
13925
13926 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13927
13928 ins_encode %{
13929 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13930 %}
13931
13932 ins_pipe(pipe_class_memory);
13933
13934 %}
13935
13936 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13937
13938 match(Set dst (MoveF2I src));
13939
13940 effect(DEF dst, USE src);
13941
13942 ins_cost(INSN_COST);
13943
13944 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13945
13946 ins_encode %{
13947 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13948 %}
13949
13950 ins_pipe(pipe_class_memory);
13951
13952 %}
13953
13954 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13955
13956 match(Set dst (MoveI2F src));
13957
13958 effect(DEF dst, USE src);
13959
13960 ins_cost(INSN_COST);
13961
13962 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13963
13964 ins_encode %{
13965 __ strw($src$$Register, Address(sp, $dst$$disp));
13966 %}
13967
13968 ins_pipe(istore_reg_reg);
13969
13970 %}
13971
13972 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13973
13974 match(Set dst (MoveD2L src));
13975
13976 effect(DEF dst, USE src);
13977
13978 ins_cost(INSN_COST);
13979
13980 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13981
13982 ins_encode %{
13983 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13984 %}
13985
13986 ins_pipe(pipe_class_memory);
13987
13988 %}
13989
13990 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
13991
13992 match(Set dst (MoveL2D src));
13993
13994 effect(DEF dst, USE src);
13995
13996 ins_cost(INSN_COST);
13997
13998 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
13999
14000 ins_encode %{
14001 __ str($src$$Register, Address(sp, $dst$$disp));
14002 %}
14003
14004 ins_pipe(istore_reg_reg);
14005
14006 %}
14007
14008 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14009
14010 match(Set dst (MoveF2I src));
14011
14012 effect(DEF dst, USE src);
14013
14014 ins_cost(INSN_COST);
14015
14016 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14017
14018 ins_encode %{
14019 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14020 %}
14021
14022 ins_pipe(fp_f2i);
14023
14024 %}
14025
14026 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14027
14028 match(Set dst (MoveI2F src));
14029
14030 effect(DEF dst, USE src);
14031
14032 ins_cost(INSN_COST);
14033
14034 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14035
14036 ins_encode %{
14037 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14038 %}
14039
14040 ins_pipe(fp_i2f);
14041
14042 %}
14043
14044 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14045
14046 match(Set dst (MoveD2L src));
14047
14048 effect(DEF dst, USE src);
14049
14050 ins_cost(INSN_COST);
14051
14052 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14053
14054 ins_encode %{
14055 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14056 %}
14057
14058 ins_pipe(fp_d2l);
14059
14060 %}
14061
14062 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14063
14064 match(Set dst (MoveL2D src));
14065
14066 effect(DEF dst, USE src);
14067
14068 ins_cost(INSN_COST);
14069
14070 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14071
14072 ins_encode %{
14073 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14074 %}
14075
14076 ins_pipe(fp_l2d);
14077
14078 %}
14079
14080 // ============================================================================
14081 // clearing of an array
14082
14083 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14084 %{
14085 match(Set dummy (ClearArray cnt base));
14086 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14087
14088 ins_cost(4 * INSN_COST);
14089 format %{ "ClearArray $cnt, $base" %}
14090
14091 ins_encode %{
14092 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14093 if (tpc == nullptr) {
14094 ciEnv::current()->record_failure("CodeCache is full");
14095 return;
14096 }
14097 %}
14098
14099 ins_pipe(pipe_class_memory);
14100 %}
14101
14102 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14103 %{
14104 predicate((uint64_t)n->in(2)->get_long()
14105 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14106 match(Set dummy (ClearArray cnt base));
14107 effect(TEMP temp, USE_KILL base, KILL cr);
14108
14109 ins_cost(4 * INSN_COST);
14110 format %{ "ClearArray $cnt, $base" %}
14111
14112 ins_encode %{
14113 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14114 if (tpc == nullptr) {
14115 ciEnv::current()->record_failure("CodeCache is full");
14116 return;
14117 }
14118 %}
14119
14120 ins_pipe(pipe_class_memory);
14121 %}
14122
14123 // ============================================================================
14124 // Overflow Math Instructions
14125
14126 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14127 %{
14128 match(Set cr (OverflowAddI op1 op2));
14129
14130 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14131 ins_cost(INSN_COST);
14132 ins_encode %{
14133 __ cmnw($op1$$Register, $op2$$Register);
14134 %}
14135
14136 ins_pipe(icmp_reg_reg);
14137 %}
14138
14139 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14140 %{
14141 match(Set cr (OverflowAddI op1 op2));
14142
14143 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14144 ins_cost(INSN_COST);
14145 ins_encode %{
14146 __ cmnw($op1$$Register, $op2$$constant);
14147 %}
14148
14149 ins_pipe(icmp_reg_imm);
14150 %}
14151
14152 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14153 %{
14154 match(Set cr (OverflowAddL op1 op2));
14155
14156 format %{ "cmn $op1, $op2\t# overflow check long" %}
14157 ins_cost(INSN_COST);
14158 ins_encode %{
14159 __ cmn($op1$$Register, $op2$$Register);
14160 %}
14161
14162 ins_pipe(icmp_reg_reg);
14163 %}
14164
14165 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14166 %{
14167 match(Set cr (OverflowAddL op1 op2));
14168
14169 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14170 ins_cost(INSN_COST);
14171 ins_encode %{
14172 __ adds(zr, $op1$$Register, $op2$$constant);
14173 %}
14174
14175 ins_pipe(icmp_reg_imm);
14176 %}
14177
14178 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14179 %{
14180 match(Set cr (OverflowSubI op1 op2));
14181
14182 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14183 ins_cost(INSN_COST);
14184 ins_encode %{
14185 __ cmpw($op1$$Register, $op2$$Register);
14186 %}
14187
14188 ins_pipe(icmp_reg_reg);
14189 %}
14190
14191 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14192 %{
14193 match(Set cr (OverflowSubI op1 op2));
14194
14195 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14196 ins_cost(INSN_COST);
14197 ins_encode %{
14198 __ cmpw($op1$$Register, $op2$$constant);
14199 %}
14200
14201 ins_pipe(icmp_reg_imm);
14202 %}
14203
14204 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14205 %{
14206 match(Set cr (OverflowSubL op1 op2));
14207
14208 format %{ "cmp $op1, $op2\t# overflow check long" %}
14209 ins_cost(INSN_COST);
14210 ins_encode %{
14211 __ cmp($op1$$Register, $op2$$Register);
14212 %}
14213
14214 ins_pipe(icmp_reg_reg);
14215 %}
14216
14217 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14218 %{
14219 match(Set cr (OverflowSubL op1 op2));
14220
14221 format %{ "cmp $op1, $op2\t# overflow check long" %}
14222 ins_cost(INSN_COST);
14223 ins_encode %{
14224 __ subs(zr, $op1$$Register, $op2$$constant);
14225 %}
14226
14227 ins_pipe(icmp_reg_imm);
14228 %}
14229
14230 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14231 %{
14232 match(Set cr (OverflowSubI zero op1));
14233
14234 format %{ "cmpw zr, $op1\t# overflow check int" %}
14235 ins_cost(INSN_COST);
14236 ins_encode %{
14237 __ cmpw(zr, $op1$$Register);
14238 %}
14239
14240 ins_pipe(icmp_reg_imm);
14241 %}
14242
14243 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14244 %{
14245 match(Set cr (OverflowSubL zero op1));
14246
14247 format %{ "cmp zr, $op1\t# overflow check long" %}
14248 ins_cost(INSN_COST);
14249 ins_encode %{
14250 __ cmp(zr, $op1$$Register);
14251 %}
14252
14253 ins_pipe(icmp_reg_imm);
14254 %}
14255
14256 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14257 %{
14258 match(Set cr (OverflowMulI op1 op2));
14259
14260 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14261 "cmp rscratch1, rscratch1, sxtw\n\t"
14262 "movw rscratch1, #0x80000000\n\t"
14263 "cselw rscratch1, rscratch1, zr, NE\n\t"
14264 "cmpw rscratch1, #1" %}
14265 ins_cost(5 * INSN_COST);
14266 ins_encode %{
14267 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14268 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14269 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14270 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14271 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14272 %}
14273
14274 ins_pipe(pipe_slow);
14275 %}
14276
14277 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14278 %{
14279 match(If cmp (OverflowMulI op1 op2));
14280 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14281 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14282 effect(USE labl, KILL cr);
14283
14284 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14285 "cmp rscratch1, rscratch1, sxtw\n\t"
14286 "b$cmp $labl" %}
14287 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14288 ins_encode %{
14289 Label* L = $labl$$label;
14290 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14291 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14292 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14293 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14294 %}
14295
14296 ins_pipe(pipe_serial);
14297 %}
14298
14299 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14300 %{
14301 match(Set cr (OverflowMulL op1 op2));
14302
14303 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14304 "smulh rscratch2, $op1, $op2\n\t"
14305 "cmp rscratch2, rscratch1, ASR #63\n\t"
14306 "movw rscratch1, #0x80000000\n\t"
14307 "cselw rscratch1, rscratch1, zr, NE\n\t"
14308 "cmpw rscratch1, #1" %}
14309 ins_cost(6 * INSN_COST);
14310 ins_encode %{
14311 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14312 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14313 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14314 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14315 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14316 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14317 %}
14318
14319 ins_pipe(pipe_slow);
14320 %}
14321
14322 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14323 %{
14324 match(If cmp (OverflowMulL op1 op2));
14325 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14326 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14327 effect(USE labl, KILL cr);
14328
14329 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14330 "smulh rscratch2, $op1, $op2\n\t"
14331 "cmp rscratch2, rscratch1, ASR #63\n\t"
14332 "b$cmp $labl" %}
14333 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14334 ins_encode %{
14335 Label* L = $labl$$label;
14336 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14337 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14338 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14339 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14340 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14341 %}
14342
14343 ins_pipe(pipe_serial);
14344 %}
14345
14346 // ============================================================================
14347 // Compare Instructions
14348
14349 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14350 %{
14351 match(Set cr (CmpI op1 op2));
14352
14353 effect(DEF cr, USE op1, USE op2);
14354
14355 ins_cost(INSN_COST);
14356 format %{ "cmpw $op1, $op2" %}
14357
14358 ins_encode(aarch64_enc_cmpw(op1, op2));
14359
14360 ins_pipe(icmp_reg_reg);
14361 %}
14362
14363 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14364 %{
14365 match(Set cr (CmpI op1 zero));
14366
14367 effect(DEF cr, USE op1);
14368
14369 ins_cost(INSN_COST);
14370 format %{ "cmpw $op1, 0" %}
14371
14372 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14373
14374 ins_pipe(icmp_reg_imm);
14375 %}
14376
14377 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14378 %{
14379 match(Set cr (CmpI op1 op2));
14380
14381 effect(DEF cr, USE op1);
14382
14383 ins_cost(INSN_COST);
14384 format %{ "cmpw $op1, $op2" %}
14385
14386 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14387
14388 ins_pipe(icmp_reg_imm);
14389 %}
14390
14391 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14392 %{
14393 match(Set cr (CmpI op1 op2));
14394
14395 effect(DEF cr, USE op1);
14396
14397 ins_cost(INSN_COST * 2);
14398 format %{ "cmpw $op1, $op2" %}
14399
14400 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14401
14402 ins_pipe(icmp_reg_imm);
14403 %}
14404
14405 // Unsigned compare Instructions; really, same as signed compare
14406 // except it should only be used to feed an If or a CMovI which takes a
14407 // cmpOpU.
14408
14409 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14410 %{
14411 match(Set cr (CmpU op1 op2));
14412
14413 effect(DEF cr, USE op1, USE op2);
14414
14415 ins_cost(INSN_COST);
14416 format %{ "cmpw $op1, $op2\t# unsigned" %}
14417
14418 ins_encode(aarch64_enc_cmpw(op1, op2));
14419
14420 ins_pipe(icmp_reg_reg);
14421 %}
14422
14423 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14424 %{
14425 match(Set cr (CmpU op1 zero));
14426
14427 effect(DEF cr, USE op1);
14428
14429 ins_cost(INSN_COST);
14430 format %{ "cmpw $op1, #0\t# unsigned" %}
14431
14432 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14433
14434 ins_pipe(icmp_reg_imm);
14435 %}
14436
14437 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14438 %{
14439 match(Set cr (CmpU op1 op2));
14440
14441 effect(DEF cr, USE op1);
14442
14443 ins_cost(INSN_COST);
14444 format %{ "cmpw $op1, $op2\t# unsigned" %}
14445
14446 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14447
14448 ins_pipe(icmp_reg_imm);
14449 %}
14450
14451 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14452 %{
14453 match(Set cr (CmpU op1 op2));
14454
14455 effect(DEF cr, USE op1);
14456
14457 ins_cost(INSN_COST * 2);
14458 format %{ "cmpw $op1, $op2\t# unsigned" %}
14459
14460 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14461
14462 ins_pipe(icmp_reg_imm);
14463 %}
14464
14465 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14466 %{
14467 match(Set cr (CmpL op1 op2));
14468
14469 effect(DEF cr, USE op1, USE op2);
14470
14471 ins_cost(INSN_COST);
14472 format %{ "cmp $op1, $op2" %}
14473
14474 ins_encode(aarch64_enc_cmp(op1, op2));
14475
14476 ins_pipe(icmp_reg_reg);
14477 %}
14478
14479 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14480 %{
14481 match(Set cr (CmpL op1 zero));
14482
14483 effect(DEF cr, USE op1);
14484
14485 ins_cost(INSN_COST);
14486 format %{ "tst $op1" %}
14487
14488 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14489
14490 ins_pipe(icmp_reg_imm);
14491 %}
14492
14493 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14494 %{
14495 match(Set cr (CmpL op1 op2));
14496
14497 effect(DEF cr, USE op1);
14498
14499 ins_cost(INSN_COST);
14500 format %{ "cmp $op1, $op2" %}
14501
14502 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14503
14504 ins_pipe(icmp_reg_imm);
14505 %}
14506
14507 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14508 %{
14509 match(Set cr (CmpL op1 op2));
14510
14511 effect(DEF cr, USE op1);
14512
14513 ins_cost(INSN_COST * 2);
14514 format %{ "cmp $op1, $op2" %}
14515
14516 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14517
14518 ins_pipe(icmp_reg_imm);
14519 %}
14520
14521 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14522 %{
14523 match(Set cr (CmpUL op1 op2));
14524
14525 effect(DEF cr, USE op1, USE op2);
14526
14527 ins_cost(INSN_COST);
14528 format %{ "cmp $op1, $op2" %}
14529
14530 ins_encode(aarch64_enc_cmp(op1, op2));
14531
14532 ins_pipe(icmp_reg_reg);
14533 %}
14534
14535 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14536 %{
14537 match(Set cr (CmpUL op1 zero));
14538
14539 effect(DEF cr, USE op1);
14540
14541 ins_cost(INSN_COST);
14542 format %{ "tst $op1" %}
14543
14544 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14545
14546 ins_pipe(icmp_reg_imm);
14547 %}
14548
14549 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14550 %{
14551 match(Set cr (CmpUL op1 op2));
14552
14553 effect(DEF cr, USE op1);
14554
14555 ins_cost(INSN_COST);
14556 format %{ "cmp $op1, $op2" %}
14557
14558 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14559
14560 ins_pipe(icmp_reg_imm);
14561 %}
14562
14563 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14564 %{
14565 match(Set cr (CmpUL op1 op2));
14566
14567 effect(DEF cr, USE op1);
14568
14569 ins_cost(INSN_COST * 2);
14570 format %{ "cmp $op1, $op2" %}
14571
14572 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14573
14574 ins_pipe(icmp_reg_imm);
14575 %}
14576
14577 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14578 %{
14579 match(Set cr (CmpP op1 op2));
14580
14581 effect(DEF cr, USE op1, USE op2);
14582
14583 ins_cost(INSN_COST);
14584 format %{ "cmp $op1, $op2\t // ptr" %}
14585
14586 ins_encode(aarch64_enc_cmpp(op1, op2));
14587
14588 ins_pipe(icmp_reg_reg);
14589 %}
14590
14591 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14592 %{
14593 match(Set cr (CmpN op1 op2));
14594
14595 effect(DEF cr, USE op1, USE op2);
14596
14597 ins_cost(INSN_COST);
14598 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14599
14600 ins_encode(aarch64_enc_cmpn(op1, op2));
14601
14602 ins_pipe(icmp_reg_reg);
14603 %}
14604
14605 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14606 %{
14607 match(Set cr (CmpP op1 zero));
14608
14609 effect(DEF cr, USE op1, USE zero);
14610
14611 ins_cost(INSN_COST);
14612 format %{ "cmp $op1, 0\t // ptr" %}
14613
14614 ins_encode(aarch64_enc_testp(op1));
14615
14616 ins_pipe(icmp_reg_imm);
14617 %}
14618
14619 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14620 %{
14621 match(Set cr (CmpN op1 zero));
14622
14623 effect(DEF cr, USE op1, USE zero);
14624
14625 ins_cost(INSN_COST);
14626 format %{ "cmp $op1, 0\t // compressed ptr" %}
14627
14628 ins_encode(aarch64_enc_testn(op1));
14629
14630 ins_pipe(icmp_reg_imm);
14631 %}
14632
14633 // FP comparisons
14634 //
14635 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14636 // using normal cmpOp. See declaration of rFlagsReg for details.
14637
14638 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14639 %{
14640 match(Set cr (CmpF src1 src2));
14641
14642 ins_cost(3 * INSN_COST);
14643 format %{ "fcmps $src1, $src2" %}
14644
14645 ins_encode %{
14646 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14647 %}
14648
14649 ins_pipe(pipe_class_compare);
14650 %}
14651
14652 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14653 %{
14654 match(Set cr (CmpF src1 src2));
14655
14656 ins_cost(3 * INSN_COST);
14657 format %{ "fcmps $src1, 0.0" %}
14658
14659 ins_encode %{
14660 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14661 %}
14662
14663 ins_pipe(pipe_class_compare);
14664 %}
14665 // FROM HERE
14666
14667 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14668 %{
14669 match(Set cr (CmpD src1 src2));
14670
14671 ins_cost(3 * INSN_COST);
14672 format %{ "fcmpd $src1, $src2" %}
14673
14674 ins_encode %{
14675 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14676 %}
14677
14678 ins_pipe(pipe_class_compare);
14679 %}
14680
14681 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14682 %{
14683 match(Set cr (CmpD src1 src2));
14684
14685 ins_cost(3 * INSN_COST);
14686 format %{ "fcmpd $src1, 0.0" %}
14687
14688 ins_encode %{
14689 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14690 %}
14691
14692 ins_pipe(pipe_class_compare);
14693 %}
14694
14695 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14696 %{
14697 match(Set dst (CmpF3 src1 src2));
14698 effect(KILL cr);
14699
14700 ins_cost(5 * INSN_COST);
14701 format %{ "fcmps $src1, $src2\n\t"
14702 "csinvw($dst, zr, zr, eq\n\t"
14703 "csnegw($dst, $dst, $dst, lt)"
14704 %}
14705
14706 ins_encode %{
14707 Label done;
14708 FloatRegister s1 = as_FloatRegister($src1$$reg);
14709 FloatRegister s2 = as_FloatRegister($src2$$reg);
14710 Register d = as_Register($dst$$reg);
14711 __ fcmps(s1, s2);
14712 // installs 0 if EQ else -1
14713 __ csinvw(d, zr, zr, Assembler::EQ);
14714 // keeps -1 if less or unordered else installs 1
14715 __ csnegw(d, d, d, Assembler::LT);
14716 __ bind(done);
14717 %}
14718
14719 ins_pipe(pipe_class_default);
14720
14721 %}
14722
14723 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14724 %{
14725 match(Set dst (CmpD3 src1 src2));
14726 effect(KILL cr);
14727
14728 ins_cost(5 * INSN_COST);
14729 format %{ "fcmpd $src1, $src2\n\t"
14730 "csinvw($dst, zr, zr, eq\n\t"
14731 "csnegw($dst, $dst, $dst, lt)"
14732 %}
14733
14734 ins_encode %{
14735 Label done;
14736 FloatRegister s1 = as_FloatRegister($src1$$reg);
14737 FloatRegister s2 = as_FloatRegister($src2$$reg);
14738 Register d = as_Register($dst$$reg);
14739 __ fcmpd(s1, s2);
14740 // installs 0 if EQ else -1
14741 __ csinvw(d, zr, zr, Assembler::EQ);
14742 // keeps -1 if less or unordered else installs 1
14743 __ csnegw(d, d, d, Assembler::LT);
14744 __ bind(done);
14745 %}
14746 ins_pipe(pipe_class_default);
14747
14748 %}
14749
14750 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14751 %{
14752 match(Set dst (CmpF3 src1 zero));
14753 effect(KILL cr);
14754
14755 ins_cost(5 * INSN_COST);
14756 format %{ "fcmps $src1, 0.0\n\t"
14757 "csinvw($dst, zr, zr, eq\n\t"
14758 "csnegw($dst, $dst, $dst, lt)"
14759 %}
14760
14761 ins_encode %{
14762 Label done;
14763 FloatRegister s1 = as_FloatRegister($src1$$reg);
14764 Register d = as_Register($dst$$reg);
14765 __ fcmps(s1, 0.0);
14766 // installs 0 if EQ else -1
14767 __ csinvw(d, zr, zr, Assembler::EQ);
14768 // keeps -1 if less or unordered else installs 1
14769 __ csnegw(d, d, d, Assembler::LT);
14770 __ bind(done);
14771 %}
14772
14773 ins_pipe(pipe_class_default);
14774
14775 %}
14776
14777 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14778 %{
14779 match(Set dst (CmpD3 src1 zero));
14780 effect(KILL cr);
14781
14782 ins_cost(5 * INSN_COST);
14783 format %{ "fcmpd $src1, 0.0\n\t"
14784 "csinvw($dst, zr, zr, eq\n\t"
14785 "csnegw($dst, $dst, $dst, lt)"
14786 %}
14787
14788 ins_encode %{
14789 Label done;
14790 FloatRegister s1 = as_FloatRegister($src1$$reg);
14791 Register d = as_Register($dst$$reg);
14792 __ fcmpd(s1, 0.0);
14793 // installs 0 if EQ else -1
14794 __ csinvw(d, zr, zr, Assembler::EQ);
14795 // keeps -1 if less or unordered else installs 1
14796 __ csnegw(d, d, d, Assembler::LT);
14797 __ bind(done);
14798 %}
14799 ins_pipe(pipe_class_default);
14800
14801 %}
14802
14803 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14804 %{
14805 match(Set dst (CmpLTMask p q));
14806 effect(KILL cr);
14807
14808 ins_cost(3 * INSN_COST);
14809
14810 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14811 "csetw $dst, lt\n\t"
14812 "subw $dst, zr, $dst"
14813 %}
14814
14815 ins_encode %{
14816 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14817 __ csetw(as_Register($dst$$reg), Assembler::LT);
14818 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14819 %}
14820
14821 ins_pipe(ialu_reg_reg);
14822 %}
14823
14824 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14825 %{
14826 match(Set dst (CmpLTMask src zero));
14827 effect(KILL cr);
14828
14829 ins_cost(INSN_COST);
14830
14831 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14832
14833 ins_encode %{
14834 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14835 %}
14836
14837 ins_pipe(ialu_reg_shift);
14838 %}
14839
14840 // ============================================================================
14841 // Max and Min
14842
14843 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14844
14845 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14846 %{
14847 effect(DEF cr, USE src);
14848 ins_cost(INSN_COST);
14849 format %{ "cmpw $src, 0" %}
14850
14851 ins_encode %{
14852 __ cmpw($src$$Register, 0);
14853 %}
14854 ins_pipe(icmp_reg_imm);
14855 %}
14856
14857 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14858 %{
14859 match(Set dst (MinI src1 src2));
14860 ins_cost(INSN_COST * 3);
14861
14862 expand %{
14863 rFlagsReg cr;
14864 compI_reg_reg(cr, src1, src2);
14865 cmovI_reg_reg_lt(dst, src1, src2, cr);
14866 %}
14867 %}
14868
14869 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14870 %{
14871 match(Set dst (MaxI src1 src2));
14872 ins_cost(INSN_COST * 3);
14873
14874 expand %{
14875 rFlagsReg cr;
14876 compI_reg_reg(cr, src1, src2);
14877 cmovI_reg_reg_gt(dst, src1, src2, cr);
14878 %}
14879 %}
14880
14881
14882 // ============================================================================
14883 // Branch Instructions
14884
14885 // Direct Branch.
14886 instruct branch(label lbl)
14887 %{
14888 match(Goto);
14889
14890 effect(USE lbl);
14891
14892 ins_cost(BRANCH_COST);
14893 format %{ "b $lbl" %}
14894
14895 ins_encode(aarch64_enc_b(lbl));
14896
14897 ins_pipe(pipe_branch);
14898 %}
14899
14900 // Conditional Near Branch
14901 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14902 %{
14903 // Same match rule as `branchConFar'.
14904 match(If cmp cr);
14905
14906 effect(USE lbl);
14907
14908 ins_cost(BRANCH_COST);
14909 // If set to 1 this indicates that the current instruction is a
14910 // short variant of a long branch. This avoids using this
14911 // instruction in first-pass matching. It will then only be used in
14912 // the `Shorten_branches' pass.
14913 // ins_short_branch(1);
14914 format %{ "b$cmp $lbl" %}
14915
14916 ins_encode(aarch64_enc_br_con(cmp, lbl));
14917
14918 ins_pipe(pipe_branch_cond);
14919 %}
14920
14921 // Conditional Near Branch Unsigned
14922 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14923 %{
14924 // Same match rule as `branchConFar'.
14925 match(If cmp cr);
14926
14927 effect(USE lbl);
14928
14929 ins_cost(BRANCH_COST);
14930 // If set to 1 this indicates that the current instruction is a
14931 // short variant of a long branch. This avoids using this
14932 // instruction in first-pass matching. It will then only be used in
14933 // the `Shorten_branches' pass.
14934 // ins_short_branch(1);
14935 format %{ "b$cmp $lbl\t# unsigned" %}
14936
14937 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14938
14939 ins_pipe(pipe_branch_cond);
14940 %}
14941
14942 // Make use of CBZ and CBNZ. These instructions, as well as being
14943 // shorter than (cmp; branch), have the additional benefit of not
14944 // killing the flags.
14945
14946 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14947 match(If cmp (CmpI op1 op2));
14948 effect(USE labl);
14949
14950 ins_cost(BRANCH_COST);
14951 format %{ "cbw$cmp $op1, $labl" %}
14952 ins_encode %{
14953 Label* L = $labl$$label;
14954 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14955 if (cond == Assembler::EQ)
14956 __ cbzw($op1$$Register, *L);
14957 else
14958 __ cbnzw($op1$$Register, *L);
14959 %}
14960 ins_pipe(pipe_cmp_branch);
14961 %}
14962
14963 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14964 match(If cmp (CmpL op1 op2));
14965 effect(USE labl);
14966
14967 ins_cost(BRANCH_COST);
14968 format %{ "cb$cmp $op1, $labl" %}
14969 ins_encode %{
14970 Label* L = $labl$$label;
14971 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14972 if (cond == Assembler::EQ)
14973 __ cbz($op1$$Register, *L);
14974 else
14975 __ cbnz($op1$$Register, *L);
14976 %}
14977 ins_pipe(pipe_cmp_branch);
14978 %}
14979
14980 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
14981 match(If cmp (CmpP op1 op2));
14982 effect(USE labl);
14983
14984 ins_cost(BRANCH_COST);
14985 format %{ "cb$cmp $op1, $labl" %}
14986 ins_encode %{
14987 Label* L = $labl$$label;
14988 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14989 if (cond == Assembler::EQ)
14990 __ cbz($op1$$Register, *L);
14991 else
14992 __ cbnz($op1$$Register, *L);
14993 %}
14994 ins_pipe(pipe_cmp_branch);
14995 %}
14996
14997 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
14998 match(If cmp (CmpN op1 op2));
14999 effect(USE labl);
15000
15001 ins_cost(BRANCH_COST);
15002 format %{ "cbw$cmp $op1, $labl" %}
15003 ins_encode %{
15004 Label* L = $labl$$label;
15005 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15006 if (cond == Assembler::EQ)
15007 __ cbzw($op1$$Register, *L);
15008 else
15009 __ cbnzw($op1$$Register, *L);
15010 %}
15011 ins_pipe(pipe_cmp_branch);
15012 %}
15013
15014 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15015 match(If cmp (CmpP (DecodeN oop) zero));
15016 effect(USE labl);
15017
15018 ins_cost(BRANCH_COST);
15019 format %{ "cb$cmp $oop, $labl" %}
15020 ins_encode %{
15021 Label* L = $labl$$label;
15022 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15023 if (cond == Assembler::EQ)
15024 __ cbzw($oop$$Register, *L);
15025 else
15026 __ cbnzw($oop$$Register, *L);
15027 %}
15028 ins_pipe(pipe_cmp_branch);
15029 %}
15030
15031 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15032 match(If cmp (CmpU op1 op2));
15033 effect(USE labl);
15034
15035 ins_cost(BRANCH_COST);
15036 format %{ "cbw$cmp $op1, $labl" %}
15037 ins_encode %{
15038 Label* L = $labl$$label;
15039 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15040 if (cond == Assembler::EQ || cond == Assembler::LS) {
15041 __ cbzw($op1$$Register, *L);
15042 } else {
15043 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15044 __ cbnzw($op1$$Register, *L);
15045 }
15046 %}
15047 ins_pipe(pipe_cmp_branch);
15048 %}
15049
15050 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15051 match(If cmp (CmpUL op1 op2));
15052 effect(USE labl);
15053
15054 ins_cost(BRANCH_COST);
15055 format %{ "cb$cmp $op1, $labl" %}
15056 ins_encode %{
15057 Label* L = $labl$$label;
15058 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15059 if (cond == Assembler::EQ || cond == Assembler::LS) {
15060 __ cbz($op1$$Register, *L);
15061 } else {
15062 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15063 __ cbnz($op1$$Register, *L);
15064 }
15065 %}
15066 ins_pipe(pipe_cmp_branch);
15067 %}
15068
15069 // Test bit and Branch
15070
15071 // Patterns for short (< 32KiB) variants
15072 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15073 match(If cmp (CmpL op1 op2));
15074 effect(USE labl);
15075
15076 ins_cost(BRANCH_COST);
15077 format %{ "cb$cmp $op1, $labl # long" %}
15078 ins_encode %{
15079 Label* L = $labl$$label;
15080 Assembler::Condition cond =
15081 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15082 __ tbr(cond, $op1$$Register, 63, *L);
15083 %}
15084 ins_pipe(pipe_cmp_branch);
15085 ins_short_branch(1);
15086 %}
15087
15088 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15089 match(If cmp (CmpI op1 op2));
15090 effect(USE labl);
15091
15092 ins_cost(BRANCH_COST);
15093 format %{ "cb$cmp $op1, $labl # int" %}
15094 ins_encode %{
15095 Label* L = $labl$$label;
15096 Assembler::Condition cond =
15097 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15098 __ tbr(cond, $op1$$Register, 31, *L);
15099 %}
15100 ins_pipe(pipe_cmp_branch);
15101 ins_short_branch(1);
15102 %}
15103
15104 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15105 match(If cmp (CmpL (AndL op1 op2) op3));
15106 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15107 effect(USE labl);
15108
15109 ins_cost(BRANCH_COST);
15110 format %{ "tb$cmp $op1, $op2, $labl" %}
15111 ins_encode %{
15112 Label* L = $labl$$label;
15113 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15114 int bit = exact_log2_long($op2$$constant);
15115 __ tbr(cond, $op1$$Register, bit, *L);
15116 %}
15117 ins_pipe(pipe_cmp_branch);
15118 ins_short_branch(1);
15119 %}
15120
15121 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15122 match(If cmp (CmpI (AndI op1 op2) op3));
15123 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15124 effect(USE labl);
15125
15126 ins_cost(BRANCH_COST);
15127 format %{ "tb$cmp $op1, $op2, $labl" %}
15128 ins_encode %{
15129 Label* L = $labl$$label;
15130 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15131 int bit = exact_log2((juint)$op2$$constant);
15132 __ tbr(cond, $op1$$Register, bit, *L);
15133 %}
15134 ins_pipe(pipe_cmp_branch);
15135 ins_short_branch(1);
15136 %}
15137
15138 // And far variants
15139 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15140 match(If cmp (CmpL op1 op2));
15141 effect(USE labl);
15142
15143 ins_cost(BRANCH_COST);
15144 format %{ "cb$cmp $op1, $labl # long" %}
15145 ins_encode %{
15146 Label* L = $labl$$label;
15147 Assembler::Condition cond =
15148 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15149 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15150 %}
15151 ins_pipe(pipe_cmp_branch);
15152 %}
15153
15154 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15155 match(If cmp (CmpI op1 op2));
15156 effect(USE labl);
15157
15158 ins_cost(BRANCH_COST);
15159 format %{ "cb$cmp $op1, $labl # int" %}
15160 ins_encode %{
15161 Label* L = $labl$$label;
15162 Assembler::Condition cond =
15163 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15164 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15165 %}
15166 ins_pipe(pipe_cmp_branch);
15167 %}
15168
15169 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15170 match(If cmp (CmpL (AndL op1 op2) op3));
15171 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15172 effect(USE labl);
15173
15174 ins_cost(BRANCH_COST);
15175 format %{ "tb$cmp $op1, $op2, $labl" %}
15176 ins_encode %{
15177 Label* L = $labl$$label;
15178 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15179 int bit = exact_log2_long($op2$$constant);
15180 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15181 %}
15182 ins_pipe(pipe_cmp_branch);
15183 %}
15184
15185 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15186 match(If cmp (CmpI (AndI op1 op2) op3));
15187 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15188 effect(USE labl);
15189
15190 ins_cost(BRANCH_COST);
15191 format %{ "tb$cmp $op1, $op2, $labl" %}
15192 ins_encode %{
15193 Label* L = $labl$$label;
15194 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15195 int bit = exact_log2((juint)$op2$$constant);
15196 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15197 %}
15198 ins_pipe(pipe_cmp_branch);
15199 %}
15200
15201 // Test bits
15202
15203 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15204 match(Set cr (CmpL (AndL op1 op2) op3));
15205 predicate(Assembler::operand_valid_for_logical_immediate
15206 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15207
15208 ins_cost(INSN_COST);
15209 format %{ "tst $op1, $op2 # long" %}
15210 ins_encode %{
15211 __ tst($op1$$Register, $op2$$constant);
15212 %}
15213 ins_pipe(ialu_reg_reg);
15214 %}
15215
15216 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15217 match(Set cr (CmpI (AndI op1 op2) op3));
15218 predicate(Assembler::operand_valid_for_logical_immediate
15219 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15220
15221 ins_cost(INSN_COST);
15222 format %{ "tst $op1, $op2 # int" %}
15223 ins_encode %{
15224 __ tstw($op1$$Register, $op2$$constant);
15225 %}
15226 ins_pipe(ialu_reg_reg);
15227 %}
15228
15229 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15230 match(Set cr (CmpL (AndL op1 op2) op3));
15231
15232 ins_cost(INSN_COST);
15233 format %{ "tst $op1, $op2 # long" %}
15234 ins_encode %{
15235 __ tst($op1$$Register, $op2$$Register);
15236 %}
15237 ins_pipe(ialu_reg_reg);
15238 %}
15239
15240 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15241 match(Set cr (CmpI (AndI op1 op2) op3));
15242
15243 ins_cost(INSN_COST);
15244 format %{ "tstw $op1, $op2 # int" %}
15245 ins_encode %{
15246 __ tstw($op1$$Register, $op2$$Register);
15247 %}
15248 ins_pipe(ialu_reg_reg);
15249 %}
15250
15251
15252 // Conditional Far Branch
15253 // Conditional Far Branch Unsigned
15254 // TODO: fixme
15255
15256 // counted loop end branch near
15257 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15258 %{
15259 match(CountedLoopEnd cmp cr);
15260
15261 effect(USE lbl);
15262
15263 ins_cost(BRANCH_COST);
15264 // short variant.
15265 // ins_short_branch(1);
15266 format %{ "b$cmp $lbl \t// counted loop end" %}
15267
15268 ins_encode(aarch64_enc_br_con(cmp, lbl));
15269
15270 ins_pipe(pipe_branch);
15271 %}
15272
15273 // counted loop end branch far
15274 // TODO: fixme
15275
15276 // ============================================================================
15277 // inlined locking and unlocking
15278
15279 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15280 %{
15281 match(Set cr (FastLock object box));
15282 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15283
15284 ins_cost(5 * INSN_COST);
15285 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15286
15287 ins_encode %{
15288 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15289 %}
15290
15291 ins_pipe(pipe_serial);
15292 %}
15293
15294 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15295 %{
15296 match(Set cr (FastUnlock object box));
15297 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15298
15299 ins_cost(5 * INSN_COST);
15300 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15301
15302 ins_encode %{
15303 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15304 %}
15305
15306 ins_pipe(pipe_serial);
15307 %}
15308
15309 // ============================================================================
15310 // Safepoint Instructions
15311
15312 // TODO
15313 // provide a near and far version of this code
15314
15315 instruct safePoint(rFlagsReg cr, iRegP poll)
15316 %{
15317 match(SafePoint poll);
15318 effect(KILL cr);
15319
15320 format %{
15321 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15322 %}
15323 ins_encode %{
15324 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15325 %}
15326 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15327 %}
15328
15329
15330 // ============================================================================
15331 // Procedure Call/Return Instructions
15332
15333 // Call Java Static Instruction
15334
15335 instruct CallStaticJavaDirect(method meth)
15336 %{
15337 match(CallStaticJava);
15338
15339 effect(USE meth);
15340
15341 ins_cost(CALL_COST);
15342
15343 format %{ "call,static $meth \t// ==> " %}
15344
15345 ins_encode(aarch64_enc_java_static_call(meth),
15346 aarch64_enc_call_epilog);
15347
15348 ins_pipe(pipe_class_call);
15349 %}
15350
15351 // TO HERE
15352
15353 // Call Java Dynamic Instruction
15354 instruct CallDynamicJavaDirect(method meth)
15355 %{
15356 match(CallDynamicJava);
15357
15358 effect(USE meth);
15359
15360 ins_cost(CALL_COST);
15361
15362 format %{ "CALL,dynamic $meth \t// ==> " %}
15363
15364 ins_encode(aarch64_enc_java_dynamic_call(meth),
15365 aarch64_enc_call_epilog);
15366
15367 ins_pipe(pipe_class_call);
15368 %}
15369
15370 // Call Runtime Instruction
15371
15372 instruct CallRuntimeDirect(method meth)
15373 %{
15374 match(CallRuntime);
15375
15376 effect(USE meth);
15377
15378 ins_cost(CALL_COST);
15379
15380 format %{ "CALL, runtime $meth" %}
15381
15382 ins_encode( aarch64_enc_java_to_runtime(meth) );
15383
15384 ins_pipe(pipe_class_call);
15385 %}
15386
15387 // Call Runtime Instruction
15388
15389 instruct CallLeafDirect(method meth)
15390 %{
15391 match(CallLeaf);
15392
15393 effect(USE meth);
15394
15395 ins_cost(CALL_COST);
15396
15397 format %{ "CALL, runtime leaf $meth" %}
15398
15399 ins_encode( aarch64_enc_java_to_runtime(meth) );
15400
15401 ins_pipe(pipe_class_call);
15402 %}
15403
15404 // Call Runtime Instruction without safepoint and with vector arguments
15405 instruct CallLeafDirectVector(method meth)
15406 %{
15407 match(CallLeafVector);
15408
15409 effect(USE meth);
15410
15411 ins_cost(CALL_COST);
15412
15413 format %{ "CALL, runtime leaf vector $meth" %}
15414
15415 ins_encode(aarch64_enc_java_to_runtime(meth));
15416
15417 ins_pipe(pipe_class_call);
15418 %}
15419
15420 // Call Runtime Instruction
15421
15422 instruct CallLeafNoFPDirect(method meth)
15423 %{
15424 match(CallLeafNoFP);
15425
15426 effect(USE meth);
15427
15428 ins_cost(CALL_COST);
15429
15430 format %{ "CALL, runtime leaf nofp $meth" %}
15431
15432 ins_encode( aarch64_enc_java_to_runtime(meth) );
15433
15434 ins_pipe(pipe_class_call);
15435 %}
15436
15437 // Tail Call; Jump from runtime stub to Java code.
15438 // Also known as an 'interprocedural jump'.
15439 // Target of jump will eventually return to caller.
15440 // TailJump below removes the return address.
15441 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15442 // emitted just above the TailCall which has reset rfp to the caller state.
15443 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15444 %{
15445 match(TailCall jump_target method_ptr);
15446
15447 ins_cost(CALL_COST);
15448
15449 format %{ "br $jump_target\t# $method_ptr holds method" %}
15450
15451 ins_encode(aarch64_enc_tail_call(jump_target));
15452
15453 ins_pipe(pipe_class_call);
15454 %}
15455
15456 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15457 %{
15458 match(TailJump jump_target ex_oop);
15459
15460 ins_cost(CALL_COST);
15461
15462 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15463
15464 ins_encode(aarch64_enc_tail_jmp(jump_target));
15465
15466 ins_pipe(pipe_class_call);
15467 %}
15468
15469 // Forward exception.
15470 instruct ForwardExceptionjmp()
15471 %{
15472 match(ForwardException);
15473 ins_cost(CALL_COST);
15474
15475 format %{ "b forward_exception_stub" %}
15476 ins_encode %{
15477 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15478 %}
15479 ins_pipe(pipe_class_call);
15480 %}
15481
15482 // Create exception oop: created by stack-crawling runtime code.
15483 // Created exception is now available to this handler, and is setup
15484 // just prior to jumping to this handler. No code emitted.
15485 // TODO check
15486 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15487 instruct CreateException(iRegP_R0 ex_oop)
15488 %{
15489 match(Set ex_oop (CreateEx));
15490
15491 format %{ " -- \t// exception oop; no code emitted" %}
15492
15493 size(0);
15494
15495 ins_encode( /*empty*/ );
15496
15497 ins_pipe(pipe_class_empty);
15498 %}
15499
15500 // Rethrow exception: The exception oop will come in the first
15501 // argument position. Then JUMP (not call) to the rethrow stub code.
15502 instruct RethrowException() %{
15503 match(Rethrow);
15504 ins_cost(CALL_COST);
15505
15506 format %{ "b rethrow_stub" %}
15507
15508 ins_encode( aarch64_enc_rethrow() );
15509
15510 ins_pipe(pipe_class_call);
15511 %}
15512
15513
15514 // Return Instruction
15515 // epilog node loads ret address into lr as part of frame pop
15516 instruct Ret()
15517 %{
15518 match(Return);
15519
15520 format %{ "ret\t// return register" %}
15521
15522 ins_encode( aarch64_enc_ret() );
15523
15524 ins_pipe(pipe_branch);
15525 %}
15526
15527 // Die now.
15528 instruct ShouldNotReachHere() %{
15529 match(Halt);
15530
15531 ins_cost(CALL_COST);
15532 format %{ "ShouldNotReachHere" %}
15533
15534 ins_encode %{
15535 if (is_reachable()) {
15536 const char* str = __ code_string(_halt_reason);
15537 __ stop(str);
15538 }
15539 %}
15540
15541 ins_pipe(pipe_class_default);
15542 %}
15543
15544 // ============================================================================
15545 // Partial Subtype Check
15546 //
15547 // superklass array for an instance of the superklass. Set a hidden
15548 // internal cache on a hit (cache is checked with exposed code in
15549 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15550 // encoding ALSO sets flags.
15551
15552 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15553 %{
15554 match(Set result (PartialSubtypeCheck sub super));
15555 predicate(!UseSecondarySupersTable);
15556 effect(KILL cr, KILL temp);
15557
15558 ins_cost(20 * INSN_COST); // slightly larger than the next version
15559 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15560
15561 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15562
15563 opcode(0x1); // Force zero of result reg on hit
15564
15565 ins_pipe(pipe_class_memory);
15566 %}
15567
15568 // Two versions of partialSubtypeCheck, both used when we need to
15569 // search for a super class in the secondary supers array. The first
15570 // is used when we don't know _a priori_ the class being searched
15571 // for. The second, far more common, is used when we do know: this is
15572 // used for instanceof, checkcast, and any case where C2 can determine
15573 // it by constant propagation.
15574
15575 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15576 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15577 rFlagsReg cr)
15578 %{
15579 match(Set result (PartialSubtypeCheck sub super));
15580 predicate(UseSecondarySupersTable);
15581 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15582
15583 ins_cost(10 * INSN_COST); // slightly larger than the next version
15584 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15585
15586 ins_encode %{
15587 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15588 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15589 $vtemp$$FloatRegister,
15590 $result$$Register, /*L_success*/nullptr);
15591 %}
15592
15593 ins_pipe(pipe_class_memory);
15594 %}
15595
15596 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15597 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15598 rFlagsReg cr)
15599 %{
15600 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15601 predicate(UseSecondarySupersTable);
15602 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15603
15604 ins_cost(5 * INSN_COST); // smaller than the next version
15605 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15606
15607 ins_encode %{
15608 bool success = false;
15609 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15610 if (InlineSecondarySupersTest) {
15611 success =
15612 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15613 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15614 $vtemp$$FloatRegister,
15615 $result$$Register,
15616 super_klass_slot);
15617 } else {
15618 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15619 success = (call != nullptr);
15620 }
15621 if (!success) {
15622 ciEnv::current()->record_failure("CodeCache is full");
15623 return;
15624 }
15625 %}
15626
15627 ins_pipe(pipe_class_memory);
15628 %}
15629
15630 // Intrisics for String.compareTo()
15631
15632 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15633 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15634 %{
15635 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15636 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15637 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15638
15639 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15640 ins_encode %{
15641 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15642 __ string_compare($str1$$Register, $str2$$Register,
15643 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15644 $tmp1$$Register, $tmp2$$Register,
15645 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15646 %}
15647 ins_pipe(pipe_class_memory);
15648 %}
15649
15650 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15651 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15652 %{
15653 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15654 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15655 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15656
15657 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15658 ins_encode %{
15659 __ string_compare($str1$$Register, $str2$$Register,
15660 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15661 $tmp1$$Register, $tmp2$$Register,
15662 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15663 %}
15664 ins_pipe(pipe_class_memory);
15665 %}
15666
15667 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15668 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15669 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15670 %{
15671 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15672 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15673 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15674 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15675
15676 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15677 ins_encode %{
15678 __ string_compare($str1$$Register, $str2$$Register,
15679 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15680 $tmp1$$Register, $tmp2$$Register,
15681 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15682 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15683 %}
15684 ins_pipe(pipe_class_memory);
15685 %}
15686
15687 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15688 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15689 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15690 %{
15691 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15692 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15693 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15694 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15695
15696 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15697 ins_encode %{
15698 __ string_compare($str1$$Register, $str2$$Register,
15699 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15700 $tmp1$$Register, $tmp2$$Register,
15701 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15702 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15703 %}
15704 ins_pipe(pipe_class_memory);
15705 %}
15706
15707 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15708 // these string_compare variants as NEON register type for convenience so that the prototype of
15709 // string_compare can be shared with all variants.
15710
15711 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15712 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15713 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15714 pRegGov_P1 pgtmp2, rFlagsReg cr)
15715 %{
15716 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15717 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15718 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15719 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15720
15721 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15722 ins_encode %{
15723 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15724 __ string_compare($str1$$Register, $str2$$Register,
15725 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15726 $tmp1$$Register, $tmp2$$Register,
15727 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15728 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15729 StrIntrinsicNode::LL);
15730 %}
15731 ins_pipe(pipe_class_memory);
15732 %}
15733
15734 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15735 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15736 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15737 pRegGov_P1 pgtmp2, rFlagsReg cr)
15738 %{
15739 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15740 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15741 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15742 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15743
15744 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15745 ins_encode %{
15746 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15747 __ string_compare($str1$$Register, $str2$$Register,
15748 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15749 $tmp1$$Register, $tmp2$$Register,
15750 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15751 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15752 StrIntrinsicNode::LU);
15753 %}
15754 ins_pipe(pipe_class_memory);
15755 %}
15756
15757 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15758 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15759 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15760 pRegGov_P1 pgtmp2, rFlagsReg cr)
15761 %{
15762 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15763 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15764 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15765 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15766
15767 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15768 ins_encode %{
15769 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15770 __ string_compare($str1$$Register, $str2$$Register,
15771 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15772 $tmp1$$Register, $tmp2$$Register,
15773 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15774 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15775 StrIntrinsicNode::UL);
15776 %}
15777 ins_pipe(pipe_class_memory);
15778 %}
15779
15780 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15781 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15782 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15783 pRegGov_P1 pgtmp2, rFlagsReg cr)
15784 %{
15785 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15786 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15787 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15788 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15789
15790 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15791 ins_encode %{
15792 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15793 __ string_compare($str1$$Register, $str2$$Register,
15794 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15795 $tmp1$$Register, $tmp2$$Register,
15796 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15797 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15798 StrIntrinsicNode::UU);
15799 %}
15800 ins_pipe(pipe_class_memory);
15801 %}
15802
15803 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15804 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15805 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15806 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15807 %{
15808 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15809 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15810 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15811 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15812 TEMP vtmp0, TEMP vtmp1, KILL cr);
15813 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15814 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15815
15816 ins_encode %{
15817 __ string_indexof($str1$$Register, $str2$$Register,
15818 $cnt1$$Register, $cnt2$$Register,
15819 $tmp1$$Register, $tmp2$$Register,
15820 $tmp3$$Register, $tmp4$$Register,
15821 $tmp5$$Register, $tmp6$$Register,
15822 -1, $result$$Register, StrIntrinsicNode::UU);
15823 %}
15824 ins_pipe(pipe_class_memory);
15825 %}
15826
15827 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15828 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15829 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15830 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15831 %{
15832 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15833 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15834 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15835 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15836 TEMP vtmp0, TEMP vtmp1, KILL cr);
15837 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15838 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15839
15840 ins_encode %{
15841 __ string_indexof($str1$$Register, $str2$$Register,
15842 $cnt1$$Register, $cnt2$$Register,
15843 $tmp1$$Register, $tmp2$$Register,
15844 $tmp3$$Register, $tmp4$$Register,
15845 $tmp5$$Register, $tmp6$$Register,
15846 -1, $result$$Register, StrIntrinsicNode::LL);
15847 %}
15848 ins_pipe(pipe_class_memory);
15849 %}
15850
15851 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15852 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15853 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15854 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15855 %{
15856 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15857 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15858 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15859 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15860 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15861 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15862 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15863
15864 ins_encode %{
15865 __ string_indexof($str1$$Register, $str2$$Register,
15866 $cnt1$$Register, $cnt2$$Register,
15867 $tmp1$$Register, $tmp2$$Register,
15868 $tmp3$$Register, $tmp4$$Register,
15869 $tmp5$$Register, $tmp6$$Register,
15870 -1, $result$$Register, StrIntrinsicNode::UL);
15871 %}
15872 ins_pipe(pipe_class_memory);
15873 %}
15874
15875 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15876 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15877 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15878 %{
15879 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15880 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15881 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15882 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15883 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15884 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15885
15886 ins_encode %{
15887 int icnt2 = (int)$int_cnt2$$constant;
15888 __ string_indexof($str1$$Register, $str2$$Register,
15889 $cnt1$$Register, zr,
15890 $tmp1$$Register, $tmp2$$Register,
15891 $tmp3$$Register, $tmp4$$Register, zr, zr,
15892 icnt2, $result$$Register, StrIntrinsicNode::UU);
15893 %}
15894 ins_pipe(pipe_class_memory);
15895 %}
15896
15897 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15898 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15899 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15900 %{
15901 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15902 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15903 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15904 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15905 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15906 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15907
15908 ins_encode %{
15909 int icnt2 = (int)$int_cnt2$$constant;
15910 __ string_indexof($str1$$Register, $str2$$Register,
15911 $cnt1$$Register, zr,
15912 $tmp1$$Register, $tmp2$$Register,
15913 $tmp3$$Register, $tmp4$$Register, zr, zr,
15914 icnt2, $result$$Register, StrIntrinsicNode::LL);
15915 %}
15916 ins_pipe(pipe_class_memory);
15917 %}
15918
15919 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15920 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15921 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15922 %{
15923 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15924 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15925 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15926 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15927 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15928 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15929
15930 ins_encode %{
15931 int icnt2 = (int)$int_cnt2$$constant;
15932 __ string_indexof($str1$$Register, $str2$$Register,
15933 $cnt1$$Register, zr,
15934 $tmp1$$Register, $tmp2$$Register,
15935 $tmp3$$Register, $tmp4$$Register, zr, zr,
15936 icnt2, $result$$Register, StrIntrinsicNode::UL);
15937 %}
15938 ins_pipe(pipe_class_memory);
15939 %}
15940
15941 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15942 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15943 iRegINoSp tmp3, rFlagsReg cr)
15944 %{
15945 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15946 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15947 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15948 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15949
15950 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15951
15952 ins_encode %{
15953 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15954 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15955 $tmp3$$Register);
15956 %}
15957 ins_pipe(pipe_class_memory);
15958 %}
15959
15960 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15961 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15962 iRegINoSp tmp3, rFlagsReg cr)
15963 %{
15964 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15965 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
15966 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
15967 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
15968
15969 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
15970
15971 ins_encode %{
15972 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
15973 $result$$Register, $tmp1$$Register, $tmp2$$Register,
15974 $tmp3$$Register);
15975 %}
15976 ins_pipe(pipe_class_memory);
15977 %}
15978
15979 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15980 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15981 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15982 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
15983 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15984 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
15985 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
15986 ins_encode %{
15987 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
15988 $result$$Register, $ztmp1$$FloatRegister,
15989 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
15990 $ptmp$$PRegister, true /* isL */);
15991 %}
15992 ins_pipe(pipe_class_memory);
15993 %}
15994
15995 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15996 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
15997 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
15998 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
15999 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16000 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16001 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16002 ins_encode %{
16003 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16004 $result$$Register, $ztmp1$$FloatRegister,
16005 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16006 $ptmp$$PRegister, false /* isL */);
16007 %}
16008 ins_pipe(pipe_class_memory);
16009 %}
16010
16011 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16012 iRegI_R0 result, rFlagsReg cr)
16013 %{
16014 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16015 match(Set result (StrEquals (Binary str1 str2) cnt));
16016 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16017
16018 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16019 ins_encode %{
16020 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16021 __ string_equals($str1$$Register, $str2$$Register,
16022 $result$$Register, $cnt$$Register);
16023 %}
16024 ins_pipe(pipe_class_memory);
16025 %}
16026
16027 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16028 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16029 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16030 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16031 iRegP_R10 tmp, rFlagsReg cr)
16032 %{
16033 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16034 match(Set result (AryEq ary1 ary2));
16035 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16036 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16037 TEMP vtmp6, TEMP vtmp7, KILL cr);
16038
16039 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16040 ins_encode %{
16041 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16042 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16043 $result$$Register, $tmp$$Register, 1);
16044 if (tpc == nullptr) {
16045 ciEnv::current()->record_failure("CodeCache is full");
16046 return;
16047 }
16048 %}
16049 ins_pipe(pipe_class_memory);
16050 %}
16051
16052 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16053 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16054 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16055 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16056 iRegP_R10 tmp, rFlagsReg cr)
16057 %{
16058 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16059 match(Set result (AryEq ary1 ary2));
16060 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16061 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16062 TEMP vtmp6, TEMP vtmp7, KILL cr);
16063
16064 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16065 ins_encode %{
16066 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16067 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16068 $result$$Register, $tmp$$Register, 2);
16069 if (tpc == nullptr) {
16070 ciEnv::current()->record_failure("CodeCache is full");
16071 return;
16072 }
16073 %}
16074 ins_pipe(pipe_class_memory);
16075 %}
16076
16077 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16078 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16079 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16080 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16081 %{
16082 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16083 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16084 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16085
16086 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16087 ins_encode %{
16088 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16089 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16090 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16091 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16092 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16093 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16094 (BasicType)$basic_type$$constant);
16095 if (tpc == nullptr) {
16096 ciEnv::current()->record_failure("CodeCache is full");
16097 return;
16098 }
16099 %}
16100 ins_pipe(pipe_class_memory);
16101 %}
16102
16103 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16104 %{
16105 match(Set result (CountPositives ary1 len));
16106 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16107 format %{ "count positives byte[] $ary1,$len -> $result" %}
16108 ins_encode %{
16109 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16110 if (tpc == nullptr) {
16111 ciEnv::current()->record_failure("CodeCache is full");
16112 return;
16113 }
16114 %}
16115 ins_pipe( pipe_slow );
16116 %}
16117
16118 // fast char[] to byte[] compression
16119 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16120 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16121 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16122 iRegI_R0 result, rFlagsReg cr)
16123 %{
16124 match(Set result (StrCompressedCopy src (Binary dst len)));
16125 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16126 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16127
16128 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16129 ins_encode %{
16130 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16131 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16132 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16133 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16134 %}
16135 ins_pipe(pipe_slow);
16136 %}
16137
16138 // fast byte[] to char[] inflation
16139 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16140 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16141 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16142 %{
16143 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16144 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16145 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16146 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16147
16148 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16149 ins_encode %{
16150 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16151 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16152 $vtmp2$$FloatRegister, $tmp$$Register);
16153 if (tpc == nullptr) {
16154 ciEnv::current()->record_failure("CodeCache is full");
16155 return;
16156 }
16157 %}
16158 ins_pipe(pipe_class_memory);
16159 %}
16160
16161 // encode char[] to byte[] in ISO_8859_1
16162 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16163 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16164 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16165 iRegI_R0 result, rFlagsReg cr)
16166 %{
16167 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16168 match(Set result (EncodeISOArray src (Binary dst len)));
16169 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16170 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16171
16172 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16173 ins_encode %{
16174 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16175 $result$$Register, false,
16176 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16177 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16178 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16179 %}
16180 ins_pipe(pipe_class_memory);
16181 %}
16182
16183 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16184 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16185 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16186 iRegI_R0 result, rFlagsReg cr)
16187 %{
16188 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16189 match(Set result (EncodeISOArray src (Binary dst len)));
16190 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16191 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16192
16193 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16194 ins_encode %{
16195 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16196 $result$$Register, true,
16197 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16198 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16199 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16200 %}
16201 ins_pipe(pipe_class_memory);
16202 %}
16203
16204 //----------------------------- CompressBits/ExpandBits ------------------------
16205
16206 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16207 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16208 match(Set dst (CompressBits src mask));
16209 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16210 format %{ "mov $tsrc, $src\n\t"
16211 "mov $tmask, $mask\n\t"
16212 "bext $tdst, $tsrc, $tmask\n\t"
16213 "mov $dst, $tdst"
16214 %}
16215 ins_encode %{
16216 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16217 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16218 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16219 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16220 %}
16221 ins_pipe(pipe_slow);
16222 %}
16223
16224 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16225 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16226 match(Set dst (CompressBits (LoadI mem) mask));
16227 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16228 format %{ "ldrs $tsrc, $mem\n\t"
16229 "ldrs $tmask, $mask\n\t"
16230 "bext $tdst, $tsrc, $tmask\n\t"
16231 "mov $dst, $tdst"
16232 %}
16233 ins_encode %{
16234 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16235 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16236 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16237 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16238 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16239 %}
16240 ins_pipe(pipe_slow);
16241 %}
16242
16243 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16244 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16245 match(Set dst (CompressBits src mask));
16246 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16247 format %{ "mov $tsrc, $src\n\t"
16248 "mov $tmask, $mask\n\t"
16249 "bext $tdst, $tsrc, $tmask\n\t"
16250 "mov $dst, $tdst"
16251 %}
16252 ins_encode %{
16253 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16254 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16255 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16256 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16257 %}
16258 ins_pipe(pipe_slow);
16259 %}
16260
16261 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16262 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16263 match(Set dst (CompressBits (LoadL mem) mask));
16264 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16265 format %{ "ldrd $tsrc, $mem\n\t"
16266 "ldrd $tmask, $mask\n\t"
16267 "bext $tdst, $tsrc, $tmask\n\t"
16268 "mov $dst, $tdst"
16269 %}
16270 ins_encode %{
16271 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16272 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16273 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16274 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16275 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16276 %}
16277 ins_pipe(pipe_slow);
16278 %}
16279
16280 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16281 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16282 match(Set dst (ExpandBits src mask));
16283 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16284 format %{ "mov $tsrc, $src\n\t"
16285 "mov $tmask, $mask\n\t"
16286 "bdep $tdst, $tsrc, $tmask\n\t"
16287 "mov $dst, $tdst"
16288 %}
16289 ins_encode %{
16290 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16291 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16292 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16293 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16294 %}
16295 ins_pipe(pipe_slow);
16296 %}
16297
16298 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16299 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16300 match(Set dst (ExpandBits (LoadI mem) mask));
16301 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16302 format %{ "ldrs $tsrc, $mem\n\t"
16303 "ldrs $tmask, $mask\n\t"
16304 "bdep $tdst, $tsrc, $tmask\n\t"
16305 "mov $dst, $tdst"
16306 %}
16307 ins_encode %{
16308 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16309 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16310 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16311 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16312 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16313 %}
16314 ins_pipe(pipe_slow);
16315 %}
16316
16317 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16318 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16319 match(Set dst (ExpandBits src mask));
16320 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16321 format %{ "mov $tsrc, $src\n\t"
16322 "mov $tmask, $mask\n\t"
16323 "bdep $tdst, $tsrc, $tmask\n\t"
16324 "mov $dst, $tdst"
16325 %}
16326 ins_encode %{
16327 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16328 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16329 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16330 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16331 %}
16332 ins_pipe(pipe_slow);
16333 %}
16334
16335
16336 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16337 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16338 match(Set dst (ExpandBits (LoadL mem) mask));
16339 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16340 format %{ "ldrd $tsrc, $mem\n\t"
16341 "ldrd $tmask, $mask\n\t"
16342 "bdep $tdst, $tsrc, $tmask\n\t"
16343 "mov $dst, $tdst"
16344 %}
16345 ins_encode %{
16346 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16347 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16348 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16349 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16350 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16351 %}
16352 ins_pipe(pipe_slow);
16353 %}
16354
16355 //----------------------------- Reinterpret ----------------------------------
16356 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16357 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16358 match(Set dst (ReinterpretHF2S src));
16359 format %{ "reinterpretHF2S $dst, $src" %}
16360 ins_encode %{
16361 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16362 %}
16363 ins_pipe(pipe_slow);
16364 %}
16365
16366 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16367 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16368 match(Set dst (ReinterpretS2HF src));
16369 format %{ "reinterpretS2HF $dst, $src" %}
16370 ins_encode %{
16371 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16372 %}
16373 ins_pipe(pipe_slow);
16374 %}
16375
16376 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16377 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16378 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16379 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16380 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16381 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16382 // can be omitted in this pattern, resulting in -
16383 // fcvt $dst, $src // Convert float to half-precision float
16384 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16385 %{
16386 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16387 format %{ "convF2HFAndS2HF $dst, $src" %}
16388 ins_encode %{
16389 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16390 %}
16391 ins_pipe(pipe_slow);
16392 %}
16393
16394 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16395 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16396 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16397 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16398 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16399 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16400 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16401 // resulting in -
16402 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16403 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16404 %{
16405 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16406 format %{ "convHF2SAndHF2F $dst, $src" %}
16407 ins_encode %{
16408 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16409 %}
16410 ins_pipe(pipe_slow);
16411 %}
16412
16413 // ============================================================================
16414 // This name is KNOWN by the ADLC and cannot be changed.
16415 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16416 // for this guy.
16417 instruct tlsLoadP(thread_RegP dst)
16418 %{
16419 match(Set dst (ThreadLocal));
16420
16421 ins_cost(0);
16422
16423 format %{ " -- \t// $dst=Thread::current(), empty" %}
16424
16425 size(0);
16426
16427 ins_encode( /*empty*/ );
16428
16429 ins_pipe(pipe_class_empty);
16430 %}
16431
16432 //----------PEEPHOLE RULES-----------------------------------------------------
16433 // These must follow all instruction definitions as they use the names
16434 // defined in the instructions definitions.
16435 //
16436 // peepmatch ( root_instr_name [preceding_instruction]* );
16437 //
16438 // peepconstraint %{
16439 // (instruction_number.operand_name relational_op instruction_number.operand_name
16440 // [, ...] );
16441 // // instruction numbers are zero-based using left to right order in peepmatch
16442 //
16443 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16444 // // provide an instruction_number.operand_name for each operand that appears
16445 // // in the replacement instruction's match rule
16446 //
16447 // ---------VM FLAGS---------------------------------------------------------
16448 //
16449 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16450 //
16451 // Each peephole rule is given an identifying number starting with zero and
16452 // increasing by one in the order seen by the parser. An individual peephole
16453 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16454 // on the command-line.
16455 //
16456 // ---------CURRENT LIMITATIONS----------------------------------------------
16457 //
16458 // Only match adjacent instructions in same basic block
16459 // Only equality constraints
16460 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16461 // Only one replacement instruction
16462 //
16463 // ---------EXAMPLE----------------------------------------------------------
16464 //
16465 // // pertinent parts of existing instructions in architecture description
16466 // instruct movI(iRegINoSp dst, iRegI src)
16467 // %{
16468 // match(Set dst (CopyI src));
16469 // %}
16470 //
16471 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16472 // %{
16473 // match(Set dst (AddI dst src));
16474 // effect(KILL cr);
16475 // %}
16476 //
16477 // // Change (inc mov) to lea
16478 // peephole %{
16479 // // increment preceded by register-register move
16480 // peepmatch ( incI_iReg movI );
16481 // // require that the destination register of the increment
16482 // // match the destination register of the move
16483 // peepconstraint ( 0.dst == 1.dst );
16484 // // construct a replacement instruction that sets
16485 // // the destination to ( move's source register + one )
16486 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16487 // %}
16488 //
16489
16490 // Implementation no longer uses movX instructions since
16491 // machine-independent system no longer uses CopyX nodes.
16492 //
16493 // peephole
16494 // %{
16495 // peepmatch (incI_iReg movI);
16496 // peepconstraint (0.dst == 1.dst);
16497 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16498 // %}
16499
16500 // peephole
16501 // %{
16502 // peepmatch (decI_iReg movI);
16503 // peepconstraint (0.dst == 1.dst);
16504 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16505 // %}
16506
16507 // peephole
16508 // %{
16509 // peepmatch (addI_iReg_imm movI);
16510 // peepconstraint (0.dst == 1.dst);
16511 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16512 // %}
16513
16514 // peephole
16515 // %{
16516 // peepmatch (incL_iReg movL);
16517 // peepconstraint (0.dst == 1.dst);
16518 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16519 // %}
16520
16521 // peephole
16522 // %{
16523 // peepmatch (decL_iReg movL);
16524 // peepconstraint (0.dst == 1.dst);
16525 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16526 // %}
16527
16528 // peephole
16529 // %{
16530 // peepmatch (addL_iReg_imm movL);
16531 // peepconstraint (0.dst == 1.dst);
16532 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16533 // %}
16534
16535 // peephole
16536 // %{
16537 // peepmatch (addP_iReg_imm movP);
16538 // peepconstraint (0.dst == 1.dst);
16539 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16540 // %}
16541
16542 // // Change load of spilled value to only a spill
16543 // instruct storeI(memory mem, iRegI src)
16544 // %{
16545 // match(Set mem (StoreI mem src));
16546 // %}
16547 //
16548 // instruct loadI(iRegINoSp dst, memory mem)
16549 // %{
16550 // match(Set dst (LoadI mem));
16551 // %}
16552 //
16553
16554 //----------SMARTSPILL RULES---------------------------------------------------
16555 // These must follow all instruction definitions as they use the names
16556 // defined in the instructions definitions.
16557
16558 // Local Variables:
16559 // mode: c++
16560 // End: